Merge branch 'trunk' into HDFS-7240

hadoop-common-project/hadoop-auth/pom.xml

@@ -135,6 +135,48 @@
       <groupId>org.apache.kerby</groupId>
       <artifactId>kerb-simplekdc</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.directory.server</groupId>
+      <artifactId>apacheds-core</artifactId>
+      <version>${apacheds.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.directory.server</groupId>
+      <artifactId>apacheds-protocol-ldap</artifactId>
+      <version>${apacheds.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.directory.server</groupId>
+      <artifactId>apacheds-ldif-partition</artifactId>
+      <version>${apacheds.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.directory.api</groupId>
+      <artifactId>api-ldap-codec-core</artifactId>
+      <version>${ldap-api.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.directory.api</groupId>
+      <artifactId>api-ldap-model</artifactId>
+      <version>${ldap-api.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.directory.server</groupId>
+      <artifactId>apacheds-server-integ</artifactId>
+      <version>${apacheds.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.directory.server</groupId>
+      <artifactId>apacheds-core-integ</artifactId>
+      <version>${apacheds.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java

@@ -14,6 +14,7 @@
 package org.apache.hadoop.security.authentication.client;
 
 import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.security.authentication.server.HttpConstants;
 import org.apache.hadoop.security.authentication.util.AuthToken;
 import org.apache.hadoop.security.authentication.util.KerberosUtil;
 import org.ietf.jgss.GSSContext;
@@ -57,17 +58,18 @@ public class KerberosAuthenticator implements Authenticator {
   /**
    * HTTP header used by the SPNEGO server endpoint during an authentication sequence.
    */
-  public static final String WWW_AUTHENTICATE = "WWW-Authenticate";
+  public static final String WWW_AUTHENTICATE =
+      HttpConstants.WWW_AUTHENTICATE_HEADER;
 
   /**
    * HTTP header used by the SPNEGO client endpoint during an authentication sequence.
    */
-  public static final String AUTHORIZATION = "Authorization";
+  public static final String AUTHORIZATION = HttpConstants.AUTHORIZATION_HEADER;
 
   /**
    * HTTP header prefix used by the SPNEGO client/server endpoints during an authentication sequence.
    */
-  public static final String NEGOTIATE = "Negotiate";
+  public static final String NEGOTIATE = HttpConstants.NEGOTIATE;
 
   private static final String AUTH_HTTP_METHOD = "OPTIONS";
 

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java

@@ -165,15 +165,9 @@ public class AuthenticationFilter implements Filter {
           PseudoAuthenticationHandler.TYPE + "|" + 
           KerberosAuthenticationHandler.TYPE + "|<class>");
     }
-    if (authHandlerName.toLowerCase(Locale.ENGLISH).equals(
-        PseudoAuthenticationHandler.TYPE)) {
-      authHandlerClassName = PseudoAuthenticationHandler.class.getName();
-    } else if (authHandlerName.toLowerCase(Locale.ENGLISH).equals(
-        KerberosAuthenticationHandler.TYPE)) {
-      authHandlerClassName = KerberosAuthenticationHandler.class.getName();
-    } else {
-      authHandlerClassName = authHandlerName;
-    }
+    authHandlerClassName =
+        AuthenticationHandlerUtil
+            .getAuthenticationHandlerClassName(authHandlerName);
     maxInactiveInterval = Long.parseLong(config.getProperty(
         AUTH_TOKEN_MAX_INACTIVE_INTERVAL, "-1")); // By default, disable.
     if (maxInactiveInterval > 0) {
@@ -452,7 +446,8 @@ public class AuthenticationFilter implements Filter {
     }
     if (tokenStr != null) {
       token = AuthenticationToken.parse(tokenStr);
-      if (!token.getType().equals(authHandler.getType())) {
+      boolean match = verifyTokenType(getAuthenticationHandler(), token);
+      if (!match) {
         throw new AuthenticationException("Invalid AuthenticationToken type");
       }
       if (token.isExpired()) {
@@ -462,6 +457,38 @@ public class AuthenticationFilter implements Filter {
     return token;
   }
 
+  /**
+   * This method verifies if the specified token type matches one of the the
+   * token types supported by a specified {@link AuthenticationHandler}. This
+   * method is specifically designed to work with
+   * {@link CompositeAuthenticationHandler} implementation which supports
+   * multiple authentication schemes while the {@link AuthenticationHandler}
+   * interface supports a single type via
+   * {@linkplain AuthenticationHandler#getType()} method.
+   *
+   * @param handler The authentication handler whose supported token types
+   *                should be used for verification.
+   * @param token   The token whose type needs to be verified.
+   * @return true   If the token type matches one of the supported token types
+   *         false  Otherwise
+   */
+  protected boolean verifyTokenType(AuthenticationHandler handler,
+      AuthenticationToken token) {
+    if(!(handler instanceof CompositeAuthenticationHandler)) {
+      return handler.getType().equals(token.getType());
+    }
+    boolean match = false;
+    Collection<String> tokenTypes =
+        ((CompositeAuthenticationHandler) handler).getTokenTypes();
+    for (String tokenType : tokenTypes) {
+      if (tokenType.equals(token.getType())) {
+        match = true;
+        break;
+      }
+    }
+    return match;
+  }
+
   /**
    * If the request has a valid authentication token it allows the request to continue to the target resource,
    * otherwise it triggers an authentication sequence using the configured {@link AuthenticationHandler}.

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java

@@ -29,7 +29,7 @@ import java.util.Properties;
  */
 public interface AuthenticationHandler {
 
-  public static final String WWW_AUTHENTICATE = "WWW-Authenticate";
+  String WWW_AUTHENTICATE = HttpConstants.WWW_AUTHENTICATE_HEADER;
 
   /**
    * Returns the authentication type of the authentication handler.

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandlerUtil.java

@@ -0,0 +1,105 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package org.apache.hadoop.security.authentication.server;
+
+import static org.apache.hadoop.security.authentication.server.HttpConstants.NEGOTIATE;
+import static org.apache.hadoop.security.authentication.server.HttpConstants.BASIC;
+import static org.apache.hadoop.security.authentication.server.HttpConstants.DIGEST;
+
+import java.util.Locale;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * This is a utility class designed to provide functionality related to
+ * {@link AuthenticationHandler}.
+ */
+public final class AuthenticationHandlerUtil {
+
+  /**
+   * This class should only contain the static utility methods. Hence it is not
+   * intended to be instantiated.
+   */
+  private AuthenticationHandlerUtil() {
+  }
+
+  /**
+   * This method provides an instance of {@link AuthenticationHandler} based on
+   * specified <code>authHandlerName</code>.
+   *
+   * @param authHandler The short-name (or fully qualified class name) of the
+   *          authentication handler.
+   * @return an instance of AuthenticationHandler implementation.
+   */
+  public static String getAuthenticationHandlerClassName(String authHandler) {
+    String handlerName =
+        Preconditions.checkNotNull(authHandler).toLowerCase(Locale.ENGLISH);
+
+    String authHandlerClassName = null;
+
+    if (handlerName.equals(PseudoAuthenticationHandler.TYPE)) {
+      authHandlerClassName = PseudoAuthenticationHandler.class.getName();
+    } else if (handlerName.equals(KerberosAuthenticationHandler.TYPE)) {
+      authHandlerClassName = KerberosAuthenticationHandler.class.getName();
+    } else if (handlerName.equals(LdapAuthenticationHandler.TYPE)) {
+      authHandlerClassName = LdapAuthenticationHandler.class.getName();
+    } else if (handlerName.equals(MultiSchemeAuthenticationHandler.TYPE)) {
+      authHandlerClassName = MultiSchemeAuthenticationHandler.class.getName();
+    } else {
+      authHandlerClassName = authHandler;
+    }
+
+    return authHandlerClassName;
+  }
+
+  /**
+   * This method checks if the specified HTTP authentication <code>scheme</code>
+   * value is valid.
+   *
+   * @param scheme HTTP authentication scheme to be checked
+   * @return Canonical representation of HTTP authentication scheme
+   * @throws IllegalArgumentException In case the specified value is not a valid
+   *           HTTP authentication scheme.
+   */
+  public static String checkAuthScheme(String scheme) {
+    if (BASIC.equalsIgnoreCase(scheme)) {
+      return BASIC;
+    } else if (NEGOTIATE.equalsIgnoreCase(scheme)) {
+      return NEGOTIATE;
+    } else if (DIGEST.equalsIgnoreCase(scheme)) {
+      return DIGEST;
+    }
+    throw new IllegalArgumentException(String.format(
+        "Unsupported HTTP authentication scheme %s ."
+            + " Supported schemes are [%s, %s, %s]", scheme, BASIC, NEGOTIATE,
+        DIGEST));
+  }
+
+  /**
+   * This method checks if the specified <code>authToken</code> belongs to the
+   * specified HTTP authentication <code>scheme</code>.
+   *
+   * @param scheme HTTP authentication scheme to be checked
+   * @param auth Authentication header value which is to be compared with the
+   *          authentication scheme.
+   * @return true If the authentication header value corresponds to the
+   *         specified authentication scheme false Otherwise.
+   */
+  public static boolean matchAuthScheme(String scheme, String auth) {
+    scheme = Preconditions.checkNotNull(scheme).trim();
+    auth = Preconditions.checkNotNull(auth).trim();
+    return auth.regionMatches(true, 0, scheme, 0, scheme.length());
+  }
+}

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/CompositeAuthenticationHandler.java

@@ -0,0 +1,30 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.apache.hadoop.security.authentication.server;
+
+import java.util.Collection;
+
+/**
+ * Interface to support multiple authentication mechanisms simultaneously.
+ *
+ */
+public interface CompositeAuthenticationHandler extends AuthenticationHandler {
+  /**
+   * This method returns the token types supported by this authentication
+   * handler.
+   *
+   * @return the token types supported by this authentication handler.
+   */
+  Collection<String> getTokenTypes();
+}

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/HttpConstants.java

@@ -0,0 +1,55 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.apache.hadoop.security.authentication.server;
+
+/**
+ * This class defines constants used for HTTP protocol entities (such as
+ * headers, methods and their values).
+ */
+public final class HttpConstants {
+
+  /**
+   * This class defines the HTTP protocol constants. Hence it is not intended
+   * to be instantiated.
+   */
+  private HttpConstants() {
+  }
+
+  /**
+   * HTTP header used by the server endpoint during an authentication sequence.
+   */
+  public static final String WWW_AUTHENTICATE_HEADER = "WWW-Authenticate";
+
+  /**
+   * HTTP header used by the client endpoint during an authentication sequence.
+   */
+  public static final String AUTHORIZATION_HEADER = "Authorization";
+
+  /**
+   * HTTP header prefix used by the SPNEGO client/server endpoints during an
+   * authentication sequence.
+   */
+  public static final String NEGOTIATE = "Negotiate";
+
+  /**
+   * HTTP header prefix used during the Basic authentication sequence.
+   */
+  public static final String BASIC = "Basic";
+
+  /**
+   * HTTP header prefix used during the Basic authentication sequence.
+   */
+  public static final String DIGEST = "Digest";
+
+}

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/LdapAuthenticationHandler.java

@@ -0,0 +1,339 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.apache.hadoop.security.authentication.server;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Hashtable;
+import java.util.Properties;
+
+import javax.naming.Context;
+import javax.naming.NamingException;
+import javax.naming.directory.InitialDirContext;
+import javax.naming.ldap.InitialLdapContext;
+import javax.naming.ldap.LdapContext;
+import javax.naming.ldap.StartTlsRequest;
+import javax.naming.ldap.StartTlsResponse;
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.SSLSession;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+/**
+ * The {@link LdapAuthenticationHandler} implements the BASIC authentication
+ * mechanism for HTTP using LDAP back-end.
+ *
+ * The supported configuration properties are:
+ * <ul>
+ * <li>ldap.providerurl: The url of the LDAP server. It does not have a default
+ * value.</li>
+ * <li>ldap.basedn: the base distinguished name (DN) to be used with the LDAP
+ * server. This value is appended to the provided user id for authentication
+ * purpose. It does not have a default value.</li>
+ * <li>ldap.binddomain: the LDAP bind domain value to be used with the LDAP
+ * server. This property is optional and useful only in case of Active
+ * Directory server.
+ * <li>ldap.enablestarttls: A boolean value used to define if the LDAP server
+ * supports 'StartTLS' extension.</li>
+ * </ul>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class LdapAuthenticationHandler implements AuthenticationHandler {
+  private static Logger logger = LoggerFactory
+      .getLogger(LdapAuthenticationHandler.class);
+
+  /**
+   * Constant that identifies the authentication mechanism.
+   */
+  public static final String TYPE = "ldap";
+
+  /**
+   * Constant that identifies the authentication mechanism to be used with the
+   * LDAP server.
+   */
+  public static final String SECURITY_AUTHENTICATION = "simple";
+
+  /**
+   * Constant for the configuration property that indicates the url of the LDAP
+   * server.
+   */
+  public static final String PROVIDER_URL = TYPE + ".providerurl";
+
+  /**
+   * Constant for the configuration property that indicates the base
+   * distinguished name (DN) to be used with the LDAP server. This value is
+   * appended to the provided user id for authentication purpose.
+   */
+  public static final String BASE_DN = TYPE + ".basedn";
+
+  /**
+   * Constant for the configuration property that indicates the LDAP bind
+   * domain value to be used with the LDAP server.
+   */
+  public static final String LDAP_BIND_DOMAIN = TYPE + ".binddomain";
+
+  /**
+   * Constant for the configuration property that indicates the base
+   * distinguished name (DN) to be used with the LDAP server. This value is
+   * appended to the provided user id for authentication purpose.
+   */
+  public static final String ENABLE_START_TLS = TYPE + ".enablestarttls";
+
+  private String ldapDomain;
+  private String baseDN;
+  private String providerUrl;
+  private Boolean enableStartTls;
+  private Boolean disableHostNameVerification;
+
+  /**
+   * Configure StartTLS LDAP extension for this handler.
+   *
+   * @param enableStartTls true If the StartTLS LDAP extension is to be enabled
+   *          false otherwise
+   */
+  @VisibleForTesting
+  public void setEnableStartTls(Boolean enableStartTls) {
+    this.enableStartTls = enableStartTls;
+  }
+
+  /**
+   * Configure the Host name verification for this handler. This method is
+   * introduced only for unit testing and should never be used in production.
+   *
+   * @param disableHostNameVerification true to disable host-name verification
+   *          false otherwise
+   */
+  @VisibleForTesting
+  public void setDisableHostNameVerification(
+      Boolean disableHostNameVerification) {
+    this.disableHostNameVerification = disableHostNameVerification;
+  }
+
+  @Override
+  public String getType() {
+    return TYPE;
+  }
+
+  @Override
+  public void init(Properties config) throws ServletException {
+    this.baseDN = config.getProperty(BASE_DN);
+    this.providerUrl = config.getProperty(PROVIDER_URL);
+    this.ldapDomain = config.getProperty(LDAP_BIND_DOMAIN);
+    this.enableStartTls =
+        Boolean.valueOf(config.getProperty(ENABLE_START_TLS, "false"));
+
+    Preconditions
+        .checkNotNull(this.providerUrl, "The LDAP URI can not be null");
+    Preconditions.checkArgument((this.baseDN == null)
+        ^ (this.ldapDomain == null),
+        "Either LDAP base DN or LDAP domain value needs to be specified");
+    if (this.enableStartTls) {
+      String tmp = this.providerUrl.toLowerCase();
+      Preconditions.checkArgument(!tmp.startsWith("ldaps"),
+          "Can not use ldaps and StartTLS option at the same time");
+    }
+  }
+
+  @Override
+  public void destroy() {
+  }
+
+  @Override
+  public boolean managementOperation(AuthenticationToken token,
+      HttpServletRequest request, HttpServletResponse response)
+      throws IOException, AuthenticationException {
+    return true;
+  }
+
+  @Override
+  public AuthenticationToken authenticate(HttpServletRequest request,
+      HttpServletResponse response)
+          throws IOException, AuthenticationException {
+    AuthenticationToken token = null;
+    String authorization =
+        request.getHeader(HttpConstants.AUTHORIZATION_HEADER);
+
+    if (authorization == null
+        || !AuthenticationHandlerUtil.matchAuthScheme(HttpConstants.BASIC,
+            authorization)) {
+      response.setHeader(WWW_AUTHENTICATE, HttpConstants.BASIC);
+      response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+      if (authorization == null) {
+        logger.trace("Basic auth starting");
+      } else {
+        logger.warn("'" + HttpConstants.AUTHORIZATION_HEADER
+            + "' does not start with '" + HttpConstants.BASIC + "' :  {}",
+            authorization);
+      }
+    } else {
+      authorization =
+          authorization.substring(HttpConstants.BASIC.length()).trim();
+      final Base64 base64 = new Base64(0);
+      // As per RFC7617, UTF-8 charset should be used for decoding.
+      String[] credentials = new String(base64.decode(authorization),
+          StandardCharsets.UTF_8).split(":", 2);
+      if (credentials.length == 2) {
+        token = authenticateUser(credentials[0], credentials[1]);
+        response.setStatus(HttpServletResponse.SC_OK);
+      }
+    }
+    return token;
+  }
+
+  private AuthenticationToken authenticateUser(String userName,
+      String password) throws AuthenticationException {
+    if (userName == null || userName.isEmpty()) {
+      throw new AuthenticationException("Error validating LDAP user:"
+          + " a null or blank username has been provided");
+    }
+
+    // If the domain is available in the config, then append it unless domain
+    // is already part of the username. LDAP providers like Active Directory
+    // use a fully qualified user name like foo@bar.com.
+    if (!hasDomain(userName) && ldapDomain != null) {
+      userName = userName + "@" + ldapDomain;
+    }
+
+    if (password == null || password.isEmpty() ||
+        password.getBytes(StandardCharsets.UTF_8)[0] == 0) {
+      throw new AuthenticationException("Error validating LDAP user:"
+          + " a null or blank password has been provided");
+    }
+
+    // setup the security principal
+    String bindDN;
+    if (baseDN == null) {
+      bindDN = userName;
+    } else {
+      bindDN = "uid=" + userName + "," + baseDN;
+    }
+
+    if (this.enableStartTls) {
+      authenticateWithTlsExtension(bindDN, password);
+    } else {
+      authenticateWithoutTlsExtension(bindDN, password);
+    }
+
+    return new AuthenticationToken(userName, userName, TYPE);
+  }
+
+  private void authenticateWithTlsExtension(String userDN, String password)
+      throws AuthenticationException {
+    LdapContext ctx = null;
+    Hashtable<String, Object> env = new Hashtable<String, Object>();
+    env.put(Context.INITIAL_CONTEXT_FACTORY,
+        "com.sun.jndi.ldap.LdapCtxFactory");
+    env.put(Context.PROVIDER_URL, providerUrl);
+
+    try {
+      // Create initial context
+      ctx = new InitialLdapContext(env, null);
+      // Establish TLS session
+      StartTlsResponse tls =
+          (StartTlsResponse) ctx.extendedOperation(new StartTlsRequest());
+
+      if (disableHostNameVerification) {
+        tls.setHostnameVerifier(new HostnameVerifier() {
+          @Override
+          public boolean verify(String hostname, SSLSession session) {
+            return true;
+          }
+        });
+      }
+
+      tls.negotiate();
+
+      // Initialize security credentials & perform read operation for
+      // verification.
+      ctx.addToEnvironment(Context.SECURITY_AUTHENTICATION,
+          SECURITY_AUTHENTICATION);
+      ctx.addToEnvironment(Context.SECURITY_PRINCIPAL, userDN);
+      ctx.addToEnvironment(Context.SECURITY_CREDENTIALS, password);
+      ctx.lookup(userDN);
+      logger.debug("Authentication successful for {}", userDN);
+
+    } catch (NamingException | IOException ex) {
+      throw new AuthenticationException("Error validating LDAP user", ex);
+    } finally {
+      if (ctx != null) {
+        try {
+          ctx.close();
+        } catch (NamingException e) { /* Ignore. */
+        }
+      }
+    }
+  }
+
+  private void authenticateWithoutTlsExtension(String userDN, String password)
+      throws AuthenticationException {
+    Hashtable<String, Object> env = new Hashtable<String, Object>();
+    env.put(Context.INITIAL_CONTEXT_FACTORY,
+        "com.sun.jndi.ldap.LdapCtxFactory");
+    env.put(Context.PROVIDER_URL, providerUrl);
+    env.put(Context.SECURITY_AUTHENTICATION, SECURITY_AUTHENTICATION);
+    env.put(Context.SECURITY_PRINCIPAL, userDN);
+    env.put(Context.SECURITY_CREDENTIALS, password);
+
+    try {
+      // Create initial context
+      Context ctx = new InitialDirContext(env);
+      ctx.close();
+      logger.debug("Authentication successful for {}", userDN);
+
+    } catch (NamingException e) {
+      throw new AuthenticationException("Error validating LDAP user", e);
+    }
+  }
+
+  private static boolean hasDomain(String userName) {
+    return (indexOfDomainMatch(userName) > 0);
+  }
+
+  /*
+   * Get the index separating the user name from domain name (the user's name
+   * up to the first '/' or '@').
+   *
+   * @param userName full user name.
+   *
+   * @return index of domain match or -1 if not found
+   */
+  private static int indexOfDomainMatch(String userName) {
+    if (userName == null) {
+      return -1;
+    }
+
+    int idx = userName.indexOf('/');
+    int idx2 = userName.indexOf('@');
+    int endIdx = Math.min(idx, idx2); // Use the earlier match.
+    // Unless at least one of '/' or '@' was not found, in
+    // which case, user the latter match.
+    if (endIdx == -1) {
+      endIdx = Math.max(idx, idx2);
+    }
+    return endIdx;
+  }
+
+}

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/MultiSchemeAuthenticationHandler.java

@@ -0,0 +1,209 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.apache.hadoop.security.authentication.server;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Properties;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+
+/**
+ * The {@link MultiSchemeAuthenticationHandler} supports configuring multiple
+ * authentication mechanisms simultaneously. e.g. server can support multiple
+ * authentication mechanisms such as Kerberos (SPENGO) and LDAP. During the
+ * authentication phase, server will specify all possible authentication schemes
+ * and let client choose the appropriate scheme. Please refer to RFC-2616 and
+ * HADOOP-12082 for more details.
+ * <p>
+ * The supported configuration properties are:
+ * <ul>
+ * <li>multi-scheme-auth-handler.schemes: A comma separated list of HTTP
+ * authentication mechanisms supported by this handler. It does not have a
+ * default value. e.g. multi-scheme-auth-handler.schemes=basic,negotiate
+ * <li>multi-scheme-auth-handler.schemes.${scheme-name}.handler: The
+ * authentication handler implementation to be used for the specified
+ * authentication scheme. It does not have a default value. e.g.
+ * multi-scheme-auth-handler.schemes.negotiate.handler=kerberos
+ * </ul>
+ *
+ * It expected that for every authentication scheme specified in
+ * multi-scheme-auth-handler.schemes property, a handler needs to be configured.
+ * Note that while scheme values in 'multi-scheme-auth-handler.schemes' property
+ * are case-insensitive, the scheme value in the handler configuration property
+ * name must be lower case. i.e. property name such as
+ * multi-scheme-auth-handler.schemes.Negotiate.handler is invalid.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class MultiSchemeAuthenticationHandler implements
+    CompositeAuthenticationHandler {
+  private static Logger logger = LoggerFactory
+      .getLogger(MultiSchemeAuthenticationHandler.class);
+  public static final String SCHEMES_PROPERTY =
+      "multi-scheme-auth-handler.schemes";
+  public static final String AUTH_HANDLER_PROPERTY =
+      "multi-scheme-auth-handler.schemes.%s.handler";
+  private static final Splitter STR_SPLITTER = Splitter.on(',').trimResults()
+      .omitEmptyStrings();
+
+  private final Map<String, AuthenticationHandler> schemeToAuthHandlerMapping =
+      new HashMap<>();
+  private final Collection<String> types = new HashSet<>();
+  private final String authType;
+
+  /**
+   * Constant that identifies the authentication mechanism.
+   */
+  public static final String TYPE = "multi-scheme";
+
+  public MultiSchemeAuthenticationHandler() {
+    this(TYPE);
+  }
+
+  public MultiSchemeAuthenticationHandler(String authType) {
+    this.authType = authType;
+  }
+
+  @Override
+  public String getType() {
+    return authType;
+  }
+
+  /**
+   * This method returns the token types supported by this authentication
+   * handler.
+   *
+   * @return the token types supported by this authentication handler.
+   */
+  @Override
+  public Collection<String> getTokenTypes() {
+    return types;
+  }
+
+  @Override
+  public void init(Properties config) throws ServletException {
+    // Useful for debugging purpose.
+    for (Map.Entry prop : config.entrySet()) {
+      logger.info("{} : {}", prop.getKey(), prop.getValue());
+    }
+
+    this.types.clear();
+
+    String schemesProperty =
+        Preconditions.checkNotNull(config.getProperty(SCHEMES_PROPERTY),
+            "%s system property is not specified.", SCHEMES_PROPERTY);
+    for (String scheme : STR_SPLITTER.split(schemesProperty)) {
+      scheme = AuthenticationHandlerUtil.checkAuthScheme(scheme);
+      if (schemeToAuthHandlerMapping.containsKey(scheme)) {
+        throw new IllegalArgumentException("Handler is already specified for "
+            + scheme + " authentication scheme.");
+      }
+
+      String authHandlerPropName =
+          String.format(AUTH_HANDLER_PROPERTY, scheme).toLowerCase();
+      String authHandlerName = config.getProperty(authHandlerPropName);
+      Preconditions.checkNotNull(authHandlerName,
+          "No auth handler configured for scheme %s.", scheme);
+
+      String authHandlerClassName =
+          AuthenticationHandlerUtil
+              .getAuthenticationHandlerClassName(authHandlerName);
+      AuthenticationHandler handler =
+          initializeAuthHandler(authHandlerClassName, config);
+      schemeToAuthHandlerMapping.put(scheme, handler);
+      types.add(handler.getType());
+    }
+    logger.info("Successfully initialized MultiSchemeAuthenticationHandler");
+  }
+
+  protected AuthenticationHandler initializeAuthHandler(
+      String authHandlerClassName, Properties config) throws ServletException {
+    try {
+      Preconditions.checkNotNull(authHandlerClassName);
+      logger.debug("Initializing Authentication handler of type "
+          + authHandlerClassName);
+      Class<?> klass =
+          Thread.currentThread().getContextClassLoader()
+              .loadClass(authHandlerClassName);
+      AuthenticationHandler authHandler =
+          (AuthenticationHandler) klass.newInstance();
+      authHandler.init(config);
+      logger.info("Successfully initialized Authentication handler of type "
+          + authHandlerClassName);
+      return authHandler;
+    } catch (ClassNotFoundException | InstantiationException
+        | IllegalAccessException ex) {
+      logger.error("Failed to initialize authentication handler "
+          + authHandlerClassName, ex);
+      throw new ServletException(ex);
+    }
+  }
+
+  @Override
+  public void destroy() {
+    for (AuthenticationHandler handler : schemeToAuthHandlerMapping.values()) {
+      handler.destroy();
+    }
+  }
+
+  @Override
+  public boolean managementOperation(AuthenticationToken token,
+      HttpServletRequest request, HttpServletResponse response)
+      throws IOException, AuthenticationException {
+    return true;
+  }
+
+  @Override
+  public AuthenticationToken authenticate(HttpServletRequest request,
+      HttpServletResponse response)
+          throws IOException, AuthenticationException {
+    String authorization =
+        request.getHeader(HttpConstants.AUTHORIZATION_HEADER);
+    if (authorization != null) {
+      for (String scheme : schemeToAuthHandlerMapping.keySet()) {
+        if (AuthenticationHandlerUtil.matchAuthScheme(scheme, authorization)) {
+          AuthenticationHandler handler =
+              schemeToAuthHandlerMapping.get(scheme);
+          AuthenticationToken token = handler.authenticate(request, response);
+          logger.trace("Token generated with type {}", token.getType());
+          return token;
+        }
+      }
+    }
+
+    // Handle the case when (authorization == null) or an invalid authorization
+    // header (e.g. a header value without the scheme name).
+    response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+    for (String scheme : schemeToAuthHandlerMapping.keySet()) {
+      response.addHeader(HttpConstants.WWW_AUTHENTICATE_HEADER, scheme);
+    }
+
+    return null;
+  }
+}

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/package-info.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Provides the server-side framework for authentication.
+ */
+@InterfaceAudience.LimitedPrivate({ "HBase", "HDFS", "MapReduce" })
+@InterfaceStability.Evolving
+package org.apache.hadoop.security.authentication.server;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;

hadoop-common-project/hadoop-auth/src/site/markdown/Configuration.md

@@ -225,6 +225,143 @@ The AltKerberos authentication mechanism is a partially implemented derivative o
     </web-app>
 ```
 
+### LDAP Configuration
+
+**IMPORTANT**: A LDAP server must be configured and running. When TLS is enabled for communication with LDAP server (either via ldaps scheme or 'start TLS' extension), configure the public certificate of the LDAP server in the local truststore.
+
+The LDAP authentication mechanism uses HTTP Basic authentication scheme to verify user specified credentials against a configured LDAP (or Active
+Directory) server. The authentication filter must be configured with the following init parameters:
+
+*   `[PREFIX.]type`: The keyword `ldap`.
+
+*   `[PREFIX.]ldap.providerurl`: The url of the LDAP server.
+
+*   `[PREFIX.]ldap.basedn`: The base distinguished name (DN) to be used with the LDAP server. This value is appended to the provided user id for authentication purpose. This property is not useful in case of Active Directory server.
+
+*   `[PREFIX.]ldap.binddomain`: The LDAP bind domain value to be used with the LDAP server. This property is optional and useful only in case of Active Directory server (e.g. example.com).
+
+*   `[PREFIX.]ldap.enablestarttls`: A boolean value used to define if the LDAP server supports 'StartTLS' extension.
+
+**Example**:
+
+```xml
+    <web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
+        ...
+
+        <filter>
+            <filter-name>authFilter</filter-name>
+            <filter-class>org.apache.hadoop.security.authentication.server.AuthenticationFilter</filter-class>
+            <init-param>
+                <param-name>type</param-name>
+                <param-value>ldap</param-value>
+            </init-param>
+            <init-param>
+                <param-name>ldap.providerurl</param-name>
+                <param-value>ldap://ldap-server-host:8920</param-value>
+            </init-param>
+            <init-param>
+                <param-name>ldap.basedn</param-name>
+                <param-value>ou=users,dc=example,dc=com</param-value>
+            </init-param>
+            <init-param>
+                <param-name>ldap.enablestarttls</param-name>
+                <param-value>true</param-value>
+            </init-param>
+         </filter>
+
+        <filter-mapping>
+            <filter-name>authFilter</filter-name>
+            <url-pattern>/ldap/*</url-pattern>
+        </filter-mapping>
+
+        ...
+    </web-app>
+```
+
+### Multi-scheme Configuration
+
+**IMPORTANT**: This configuration supports multiple authentication mechanisms (e.g. kerberos, ldap etc.) together. Please refer to the documentation for each individual scheme for configuration related details.
+
+The multi-scheme authentication mechanism supports multiple authentication mechanisms (e.g. kerberos, ldap etc.) by implementing a HTTP auth negotiation mechanism (Please refer to RFC-2616). For enabling each type of authentication mechanism (e.g. ldap) a corresponding authentication handler must be configured. Please refer to following configuration parameters:
+
+*   `[PREFIX.]type`: The keyword `multi-scheme`.
+
+*   `[PREFIX.]multi-scheme-auth-handler.schemes`: A comma separated list of HTTP authentication mechanisms supported by this handler. It is a required parameter and it does not have a default value (e.g. multi-scheme-auth-handler.schemes=basic,negotiate).
+
+*   `[PREFIX.]multi-scheme-auth-handler.schemes.<scheme-name>.handler`: The authentication handler implementation to be used for the specified authentication scheme. It does not have a default value (e.g. multi-scheme-auth-handler.schemes.negotiate.handler=kerberos). Add this handler configuration for each of the scheme configured.
+
+In addition to these parameters, please specify the init parameters for each handler configured as well.
+
+
+**Example**:
+
+```xml
+    <web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
+        ...
+
+        <filter>
+            <filter-name>authFilter</filter-name>
+            <filter-class>org.apache.hadoop.security.authentication.server.AuthenticationFilter</filter-class>
+            <init-param>
+                <param-name>type</param-name>
+                <param-value>multi-scheme</param-value>
+            </init-param>
+            <init-param>
+                <param-name>multi-scheme-auth-handler.schemes</param-name>
+                <param-value>basic,negotiate</param-value>
+            </init-param>
+            <init-param>
+                <param-name>multi-scheme-auth-handler.basic.handler</param-name>
+                <param-value>ldap</param-value>
+            </init-param>
+            <init-param>
+                <param-name>multi-scheme-auth-handler.negotiate.handler</param-name>
+                <param-value>kerberos</param-value>
+            </init-param>
+            <init-param>
+                <param-name>ldap.providerurl</param-name>
+                <param-value>ldap://ldap-server-host:8920</param-value>
+            </init-param>
+            <init-param>
+                <param-name>ldap.basedn</param-name>
+                <param-value>ou=users,dc=example,dc=com</param-value>
+            </init-param>
+            <init-param>
+                <param-name>ldap.enablestarttls</param-name>
+                <param-value>true</param-value>
+            </init-param>
+            <init-param>
+                <param-name>token.validity</param-name>
+                <param-value>30</param-value>
+            </init-param>
+            <init-param>
+                <param-name>cookie.domain</param-name>
+                <param-value>.foo.com</param-value>
+            </init-param>
+            <init-param>
+                <param-name>cookie.path</param-name>
+                <param-value>/</param-value>
+            </init-param>
+            <init-param>
+                <param-name>kerberos.principal</param-name>
+                <param-value>HTTP/localhost@LOCALHOST</param-value>
+            </init-param>
+            <init-param>
+                <param-name>kerberos.keytab</param-name>
+                <param-value>/tmp/auth.keytab</param-value>
+            </init-param>
+         </filter>
+
+        <filter-mapping>
+            <filter-name>authFilter</filter-name>
+            <url-pattern>/multi-scheme/*</url-pattern>
+        </filter-mapping>
+
+        ...
+    </web-app>
+```
+
+
 ### SignerSecretProvider Configuration
 
 The SignerSecretProvider is used to provide more advanced behaviors for the secret used for signing the HTTP Cookies.

hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java

@@ -20,14 +20,15 @@ import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.http.HttpResponse;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.Credentials;
+import org.apache.http.client.CredentialsProvider;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.client.params.AuthPolicy;
 import org.apache.http.entity.InputStreamEntity;
-import org.apache.http.impl.auth.SPNegoSchemeFactory;
-import org.apache.http.impl.client.SystemDefaultHttpClient;
+import org.apache.http.impl.auth.SPNegoScheme;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.util.EntityUtils;
 import org.mortbay.jetty.Server;
 import org.mortbay.jetty.servlet.Context;
@@ -53,6 +54,7 @@ import java.net.ServerSocket;
 import java.net.URL;
 import java.security.Principal;
 import java.util.Properties;
+
 import org.junit.Assert;
 
 public class AuthenticatorTestCase {
@@ -241,22 +243,29 @@ public class AuthenticatorTestCase {
     }
   }
 
-  private SystemDefaultHttpClient getHttpClient() {
-    final SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient();
-    httpClient.getAuthSchemes().register(AuthPolicy.SPNEGO, new SPNegoSchemeFactory(true));
-     Credentials use_jaas_creds = new Credentials() {
-       public String getPassword() {
-         return null;
-       }
-
-       public Principal getUserPrincipal() {
-         return null;
-       }
-     };
-
-     httpClient.getCredentialsProvider().setCredentials(
-       AuthScope.ANY, use_jaas_creds);
-     return httpClient;
+  private HttpClient getHttpClient() {
+    HttpClientBuilder builder = HttpClientBuilder.create();
+    // Register auth schema
+    builder.setDefaultAuthSchemeRegistry(
+        s-> httpContext -> new SPNegoScheme(true, true)
+    );
+
+    Credentials useJaasCreds = new Credentials() {
+      public String getPassword() {
+        return null;
+      }
+      public Principal getUserPrincipal() {
+        return null;
+      }
+    };
+
+    CredentialsProvider jaasCredentialProvider
+        = new BasicCredentialsProvider();
+    jaasCredentialProvider.setCredentials(AuthScope.ANY, useJaasCreds);
+    // Set credential provider
+    builder.setDefaultCredentialsProvider(jaasCredentialProvider);
+
+    return builder.build();
   }
 
   private void doHttpClientRequest(HttpClient httpClient, HttpUriRequest request) throws Exception {
@@ -273,7 +282,7 @@ public class AuthenticatorTestCase {
   protected void _testAuthenticationHttpClient(Authenticator authenticator, boolean doPost) throws Exception {
     start();
     try {
-      SystemDefaultHttpClient httpClient = getHttpClient();
+      HttpClient httpClient = getHttpClient();
       doHttpClientRequest(httpClient, new HttpGet(getBaseURL()));
 
       // Always do a GET before POST to trigger the SPNego negotiation

hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestKerberosAuthenticator.java

@@ -13,15 +13,22 @@
  */
 package org.apache.hadoop.security.authentication.client;
 
+import static org.apache.hadoop.security.authentication.server.MultiSchemeAuthenticationHandler.SCHEMES_PROPERTY;
+import static org.apache.hadoop.security.authentication.server.MultiSchemeAuthenticationHandler.AUTH_HANDLER_PROPERTY;
+import static org.apache.hadoop.security.authentication.server.AuthenticationFilter.AUTH_TYPE;
+import static org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler.PRINCIPAL;
+import static org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler.KEYTAB;
+import static org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler.NAME_RULES;
+
 import org.apache.hadoop.minikdc.KerberosSecurityTestcase;
 import org.apache.hadoop.security.authentication.KerberosTestUtils;
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
+import org.apache.hadoop.security.authentication.server.MultiSchemeAuthenticationHandler;
 import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
 import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
 import org.junit.runner.RunWith;
 import org.junit.Test;
 
@@ -71,6 +78,19 @@ public class TestKerberosAuthenticator extends KerberosSecurityTestcase {
     return props;
   }
 
+  private Properties getMultiAuthHandlerConfiguration() {
+    Properties props = new Properties();
+    props.setProperty(AUTH_TYPE, MultiSchemeAuthenticationHandler.TYPE);
+    props.setProperty(SCHEMES_PROPERTY, "negotiate");
+    props.setProperty(String.format(AUTH_HANDLER_PROPERTY, "negotiate"),
+        "kerberos");
+    props.setProperty(PRINCIPAL, KerberosTestUtils.getServerPrincipal());
+    props.setProperty(KEYTAB, KerberosTestUtils.getKeytabFile());
+    props.setProperty(NAME_RULES,
+        "RULE:[1:$1@$0](.*@" + KerberosTestUtils.getRealm() + ")s/@.*//\n");
+    return props;
+  }
+
   @Test(timeout=60000)
   public void testFallbacktoPseudoAuthenticator() throws Exception {
     AuthenticatorTestCase auth = new AuthenticatorTestCase(useTomcat);
@@ -162,4 +182,53 @@ public class TestKerberosAuthenticator extends KerberosSecurityTestcase {
       }
     });
   }
+
+  @Test(timeout = 60000)
+  public void testNotAuthenticatedWithMultiAuthHandler() throws Exception {
+    AuthenticatorTestCase auth = new AuthenticatorTestCase(useTomcat);
+    AuthenticatorTestCase
+        .setAuthenticationHandlerConfig(getMultiAuthHandlerConfiguration());
+    auth.start();
+    try {
+      URL url = new URL(auth.getBaseURL());
+      HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+      conn.connect();
+      Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED,
+          conn.getResponseCode());
+      Assert.assertTrue(conn
+          .getHeaderField(KerberosAuthenticator.WWW_AUTHENTICATE) != null);
+    } finally {
+      auth.stop();
+    }
+  }
+
+  @Test(timeout = 60000)
+  public void testAuthenticationWithMultiAuthHandler() throws Exception {
+    final AuthenticatorTestCase auth = new AuthenticatorTestCase(useTomcat);
+    AuthenticatorTestCase
+        .setAuthenticationHandlerConfig(getMultiAuthHandlerConfiguration());
+    KerberosTestUtils.doAsClient(new Callable<Void>() {
+      @Override
+      public Void call() throws Exception {
+        auth._testAuthentication(new KerberosAuthenticator(), false);
+        return null;
+      }
+    });
+  }
+
+  @Test(timeout = 60000)
+  public void testAuthenticationHttpClientPostWithMultiAuthHandler()
+      throws Exception {
+    final AuthenticatorTestCase auth = new AuthenticatorTestCase(useTomcat);
+    AuthenticatorTestCase
+        .setAuthenticationHandlerConfig(getMultiAuthHandlerConfiguration());
+    KerberosTestUtils.doAsClient(new Callable<Void>() {
+      @Override
+      public Void call() throws Exception {
+        auth._testAuthenticationHttpClient(new KerberosAuthenticator(), true);
+        return null;
+      }
+    });
+  }
+
 }

hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/LdapConstants.java

@@ -0,0 +1,31 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.apache.hadoop.security.authentication.server;
+
+/**
+ * This class defines the constants used by the LDAP integration tests.
+ */
+public final class LdapConstants {
+
+  /**
+   * This class defines constants to be used for LDAP integration testing.
+   * Hence this class is not expected to be instantiated.
+   */
+  private LdapConstants() {
+  }
+
+  public static final String LDAP_BASE_DN = "dc=example,dc=com";
+  public static final String LDAP_SERVER_ADDR = "localhost";
+
+}

hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestLdapAuthenticationHandler.java

@@ -0,0 +1,159 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.apache.hadoop.security.authentication.server;
+
+import java.util.Properties;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import static org.apache.hadoop.security.authentication.server.LdapAuthenticationHandler.*;
+import static org.apache.hadoop.security.authentication.server.LdapConstants.*;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.directory.server.annotations.CreateLdapServer;
+import org.apache.directory.server.annotations.CreateTransport;
+import org.apache.directory.server.core.annotations.ApplyLdifs;
+import org.apache.directory.server.core.annotations.ContextEntry;
+import org.apache.directory.server.core.annotations.CreateDS;
+import org.apache.directory.server.core.annotations.CreatePartition;
+import org.apache.directory.server.core.integ.AbstractLdapTestUnit;
+import org.apache.directory.server.core.integ.FrameworkRunner;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+
+/**
+ * This unit test verifies the functionality of LDAP authentication handler.
+ */
+@RunWith(FrameworkRunner.class)
+@CreateLdapServer(
+    transports =
+      {
+        @CreateTransport(protocol = "LDAP", address= LDAP_SERVER_ADDR),
+      })
+@CreateDS(allowAnonAccess = true,
+          partitions = {
+            @CreatePartition(
+                name = "Test_Partition", suffix = LDAP_BASE_DN,
+                contextEntry = @ContextEntry(
+                    entryLdif = "dn: " + LDAP_BASE_DN + " \n"
+                              + "dc: example\n"
+                              + "objectClass: top\n"
+                              + "objectClass: domain\n\n"))})
+@ApplyLdifs({
+    "dn: uid=bjones," + LDAP_BASE_DN,
+    "cn: Bob Jones",
+    "sn: Jones",
+    "objectClass: inetOrgPerson",
+    "uid: bjones",
+    "userPassword: p@ssw0rd"})
+public class TestLdapAuthenticationHandler extends AbstractLdapTestUnit {
+  private LdapAuthenticationHandler handler;
+
+  @Before
+  public void setup() throws Exception {
+    handler = new LdapAuthenticationHandler();
+    try {
+      handler.init(getDefaultProperties());
+    } catch (Exception e) {
+      handler = null;
+      throw e;
+    }
+  }
+
+  protected Properties getDefaultProperties() {
+    Properties p = new Properties();
+    p.setProperty(BASE_DN, LDAP_BASE_DN);
+    p.setProperty(PROVIDER_URL, String.format("ldap://%s:%s", LDAP_SERVER_ADDR,
+        getLdapServer().getPort()));
+    return p;
+  }
+
+  @Test(timeout = 60000)
+  public void testRequestWithoutAuthorization() throws Exception {
+    HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+    HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+    Assert.assertNull(handler.authenticate(request, response));
+    Mockito.verify(response).setHeader(WWW_AUTHENTICATE, HttpConstants.BASIC);
+    Mockito.verify(response).setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+  }
+
+  @Test(timeout = 60000)
+  public void testRequestWithInvalidAuthorization() throws Exception {
+    HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+    HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+    final Base64 base64 = new Base64(0);
+    String credentials = "bjones:invalidpassword";
+    Mockito.when(request.getHeader(HttpConstants.AUTHORIZATION_HEADER))
+        .thenReturn(base64.encodeToString(credentials.getBytes()));
+    Assert.assertNull(handler.authenticate(request, response));
+    Mockito.verify(response).setHeader(WWW_AUTHENTICATE, HttpConstants.BASIC);
+    Mockito.verify(response).setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+  }
+
+  @Test(timeout = 60000)
+  public void testRequestWithIncompleteAuthorization() throws Exception {
+    HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+    HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+    Mockito.when(request.getHeader(HttpConstants.AUTHORIZATION_HEADER))
+        .thenReturn(HttpConstants.BASIC);
+    Assert.assertNull(handler.authenticate(request, response));
+  }
+
+  @Test(timeout = 60000)
+  public void testRequestWithAuthorization() throws Exception {
+    HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+    HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+    final Base64 base64 = new Base64(0);
+    String credentials = base64.encodeToString("bjones:p@ssw0rd".getBytes());
+    String authHeader = HttpConstants.BASIC + " " + credentials;
+    Mockito.when(request.getHeader(HttpConstants.AUTHORIZATION_HEADER))
+        .thenReturn(authHeader);
+    AuthenticationToken token = handler.authenticate(request, response);
+    Assert.assertNotNull(token);
+    Mockito.verify(response).setStatus(HttpServletResponse.SC_OK);
+    Assert.assertEquals(TYPE, token.getType());
+    Assert.assertEquals("bjones", token.getUserName());
+    Assert.assertEquals("bjones", token.getName());
+  }
+
+  @Test(timeout = 60000)
+  public void testRequestWithWrongCredentials() throws Exception {
+    HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+    HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+    final Base64 base64 = new Base64(0);
+    String credentials = base64.encodeToString("bjones:foo123".getBytes());
+    String authHeader = HttpConstants.BASIC + " " + credentials;
+    Mockito.when(request.getHeader(HttpConstants.AUTHORIZATION_HEADER))
+        .thenReturn(authHeader);
+
+    try {
+      handler.authenticate(request, response);
+      Assert.fail();
+    } catch (AuthenticationException ex) {
+      // Expected
+    } catch (Exception ex) {
+      Assert.fail();
+    }
+  }
+}

hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestMultiSchemeAuthenticationHandler.java

@@ -0,0 +1,189 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.apache.hadoop.security.authentication.server;
+
+import static org.apache.hadoop.security.authentication.server.LdapAuthenticationHandler.BASE_DN;
+import static org.apache.hadoop.security.authentication.server.LdapAuthenticationHandler.PROVIDER_URL;
+import static org.apache.hadoop.security.authentication.server.LdapAuthenticationHandler.TYPE;
+import static org.apache.hadoop.security.authentication.server.MultiSchemeAuthenticationHandler.SCHEMES_PROPERTY;
+import static org.apache.hadoop.security.authentication.server.MultiSchemeAuthenticationHandler.AUTH_HANDLER_PROPERTY;
+import static org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler.PRINCIPAL;
+import static org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler.KEYTAB;
+import static org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler.NAME_RULES;
+import static org.apache.hadoop.security.authentication.server.LdapConstants.*;
+import static org.apache.hadoop.security.authentication.server.HttpConstants.*;
+
+import java.io.File;
+import java.util.Properties;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.directory.server.annotations.CreateLdapServer;
+import org.apache.directory.server.annotations.CreateTransport;
+import org.apache.directory.server.core.annotations.ApplyLdifs;
+import org.apache.directory.server.core.annotations.ContextEntry;
+import org.apache.directory.server.core.annotations.CreateDS;
+import org.apache.directory.server.core.annotations.CreatePartition;
+import org.apache.directory.server.core.integ.AbstractLdapTestUnit;
+import org.apache.directory.server.core.integ.FrameworkRunner;
+import org.apache.hadoop.minikdc.KerberosSecurityTestcase;
+import org.apache.hadoop.security.authentication.KerberosTestUtils;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+
+/**
+ * This unit test verifies the functionality of "multi-scheme" auth handler.
+ */
+@RunWith(FrameworkRunner.class)
+@CreateLdapServer(
+    transports =
+      {
+        @CreateTransport(protocol = "LDAP", address = LDAP_SERVER_ADDR),
+      })
+@CreateDS(allowAnonAccess = true,
+          partitions = {
+            @CreatePartition(
+              name = "Test_Partition", suffix = LDAP_BASE_DN,
+              contextEntry = @ContextEntry(
+                  entryLdif = "dn: "+ LDAP_BASE_DN+ " \n"
+                              + "dc: example\n"
+                              + "objectClass: top\n"
+                              + "objectClass: domain\n\n"))})
+@ApplyLdifs({
+    "dn: uid=bjones," + LDAP_BASE_DN,
+    "cn: Bob Jones",
+    "sn: Jones",
+    "objectClass: inetOrgPerson",
+    "uid: bjones",
+    "userPassword: p@ssw0rd"})
+public class TestMultiSchemeAuthenticationHandler
+    extends AbstractLdapTestUnit {
+  private KerberosSecurityTestcase krbTest = new KerberosSecurityTestcase();
+  private MultiSchemeAuthenticationHandler handler;
+
+  @Before
+  public void setUp()  throws Exception {
+    krbTest.startMiniKdc();
+
+    // create keytab
+    File keytabFile = new File(KerberosTestUtils.getKeytabFile());
+    String clientPrinc = KerberosTestUtils.getClientPrincipal();
+    String serverPrinc = KerberosTestUtils.getServerPrincipal();
+    clientPrinc = clientPrinc.substring(0, clientPrinc.lastIndexOf("@"));
+    serverPrinc = serverPrinc.substring(0, serverPrinc.lastIndexOf("@"));
+    krbTest.getKdc().createPrincipal(keytabFile, clientPrinc, serverPrinc);
+    // configure handler
+    handler = new MultiSchemeAuthenticationHandler();
+    try {
+      handler.init(getDefaultProperties());
+    } catch (Exception e) {
+      throw e;
+    }
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    krbTest.stopMiniKdc();
+  }
+
+  private Properties getDefaultProperties() {
+    Properties p = new Properties();
+    p.setProperty(SCHEMES_PROPERTY, BASIC + "," + NEGOTIATE);
+    p.setProperty(String.format(AUTH_HANDLER_PROPERTY, "negotiate"),
+        "kerberos");
+    p.setProperty(String.format(AUTH_HANDLER_PROPERTY, "basic"), "ldap");
+    // Kerberos related config
+    p.setProperty(PRINCIPAL, KerberosTestUtils.getServerPrincipal());
+    p.setProperty(KEYTAB, KerberosTestUtils.getKeytabFile());
+    p.setProperty(NAME_RULES,
+        "RULE:[1:$1@$0](.*@" + KerberosTestUtils.getRealm()+")s/@.*//\n");
+    // LDAP related config
+    p.setProperty(BASE_DN, LDAP_BASE_DN);
+    p.setProperty(PROVIDER_URL, String.format("ldap://%s:%s", LDAP_SERVER_ADDR,
+        getLdapServer().getPort()));
+    return p;
+  }
+
+  @Test(timeout = 60000)
+  public void testRequestWithoutAuthorization() throws Exception {
+    HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+    HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+    Assert.assertNull(handler.authenticate(request, response));
+    Mockito.verify(response).addHeader(WWW_AUTHENTICATE_HEADER, BASIC);
+    Mockito.verify(response).addHeader(WWW_AUTHENTICATE_HEADER, NEGOTIATE);
+    Mockito.verify(response).setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+  }
+
+  @Test(timeout = 60000)
+  public void testRequestWithInvalidAuthorization() throws Exception {
+    HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+    HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+    final Base64 base64 = new Base64(0);
+    String credentials = "bjones:invalidpassword";
+    Mockito.when(request.getHeader(AUTHORIZATION_HEADER))
+        .thenReturn(base64.encodeToString(credentials.getBytes()));
+    Assert.assertNull(handler.authenticate(request, response));
+    Mockito.verify(response).addHeader(WWW_AUTHENTICATE_HEADER, BASIC);
+    Mockito.verify(response).addHeader(WWW_AUTHENTICATE_HEADER, NEGOTIATE);
+    Mockito.verify(response).setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+  }
+
+  @Test(timeout = 60000)
+  public void testRequestWithLdapAuthorization() throws Exception {
+    HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+    HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+    final Base64 base64 = new Base64(0);
+    String credentials = base64.encodeToString("bjones:p@ssw0rd".getBytes());
+    String authHeader = BASIC + " " + credentials;
+    Mockito.when(request.getHeader(AUTHORIZATION_HEADER))
+        .thenReturn(authHeader);
+    AuthenticationToken token = handler.authenticate(request, response);
+    Assert.assertNotNull(token);
+    Mockito.verify(response).setStatus(HttpServletResponse.SC_OK);
+    Assert.assertEquals(TYPE, token.getType());
+    Assert.assertEquals("bjones", token.getUserName());
+    Assert.assertEquals("bjones", token.getName());
+  }
+
+  @Test(timeout = 60000)
+  public void testRequestWithInvalidKerberosAuthorization() throws Exception {
+    String token = new Base64(0).encodeToString(new byte[]{0, 1, 2});
+
+    HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+    HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+    Mockito.when(request.getHeader(AUTHORIZATION_HEADER)).thenReturn(
+        NEGOTIATE + token);
+
+    try {
+      handler.authenticate(request, response);
+      Assert.fail();
+    } catch (AuthenticationException ex) {
+      // Expected
+    } catch (Exception ex) {
+      Assert.fail();
+    }
+  }
+
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java

@@ -21,9 +21,6 @@ package org.apache.hadoop.fs;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.http.lib.StaticUserWebFilter;
-import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
-import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy;
-import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory;
 
 /** 
  * This class contains constants for configuration keys used
@@ -160,30 +157,7 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
   public static final boolean IO_COMPRESSION_CODEC_LZ4_USELZ4HC_DEFAULT =
       false;
 
-  /**
-   * Erasure Coding configuration family
-   */
 
-  /** Supported erasure codec classes */
-  public static final String IO_ERASURECODE_CODECS_KEY = "io.erasurecode.codecs";
-
-  /** Raw coder factory for the RS default codec. */
-  public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY =
-      "io.erasurecode.codec.rs-default.rawcoder";
-  public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT =
-      RSRawErasureCoderFactory.class.getCanonicalName();
-
-  /** Raw coder factory for the RS legacy codec. */
-  public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY =
-      "io.erasurecode.codec.rs-legacy.rawcoder";
-  public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT =
-      RSRawErasureCoderFactoryLegacy.class.getCanonicalName();
-
-  /** Raw coder factory for the XOR codec. */
-  public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY =
-      "io.erasurecode.codec.xor.rawcoder";
-  public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT =
-      XORRawErasureCoderFactory.class.getCanonicalName();
 
   /**
    * Service Authorization

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java

@@ -1524,13 +1524,14 @@ public abstract class FileSystem extends Configured implements Closeable {
    * <p>
    * Does not guarantee to return the List of files/directories status in a
    * sorted order.
+   * <p>
+   * Will not return null. Expect IOException upon access error.
    * @param f given path
    * @return the statuses of the files/directories in the given patch
-   * @throws FileNotFoundException when the path does not exist;
-   *         IOException see specific implementation
+   * @throws FileNotFoundException when the path does not exist
+   * @throws IOException see specific implementation
    */
-  public abstract FileStatus[] listStatus(Path f) throws FileNotFoundException, 
-                                                         IOException;
+  public abstract FileStatus[] listStatus(Path f) throws IOException;
 
   /**
    * Represents a batch of directory entries when iteratively listing a
@@ -1600,10 +1601,7 @@ public abstract class FileSystem extends Configured implements Closeable {
   private void listStatus(ArrayList<FileStatus> results, Path f,
       PathFilter filter) throws FileNotFoundException, IOException {
     FileStatus listing[] = listStatus(f);
-    if (listing == null) {
-      throw new IOException("Error accessing " + f);
-    }
-
+    Preconditions.checkNotNull(listing, "listStatus should not return NULL");
     for (int i = 0; i < listing.length; i++) {
       if (filter.accept(listing[i].getPath())) {
         results.add(listing[i]);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java

@@ -466,10 +466,7 @@ public class RawLocalFileSystem extends FileSystem {
     }
 
     if (localf.isDirectory()) {
-      String[] names = localf.list();
-      if (names == null) {
-        return null;
-      }
+      String[] names = FileUtil.list(localf);
       results = new FileStatus[names.length];
       int j = 0;
       for (int i = 0; i < names.length; i++) {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java

@@ -40,6 +40,8 @@ import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.util.Time;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /** Provides a <i>trash</i> feature.  Files are moved to a user's trash
  * directory, a subdirectory of their home directory named ".Trash".  Files are
  * initially moved to a <i>current</i> sub-directory of the trash directory.
@@ -215,7 +217,7 @@ public class TrashPolicyDefault extends TrashPolicy {
     return new Emptier(getConf(), emptierInterval);
   }
 
-  private class Emptier implements Runnable {
+  protected class Emptier implements Runnable {
 
     private Configuration conf;
     private long emptierInterval;
@@ -223,7 +225,7 @@ public class TrashPolicyDefault extends TrashPolicy {
     Emptier(Configuration conf, long emptierInterval) throws IOException {
       this.conf = conf;
       this.emptierInterval = emptierInterval;
-      if (emptierInterval > deletionInterval || emptierInterval == 0) {
+      if (emptierInterval > deletionInterval || emptierInterval <= 0) {
         LOG.info("The configured checkpoint interval is " +
                  (emptierInterval / MSECS_PER_MINUTE) + " minutes." +
                  " Using an interval of " +
@@ -287,6 +289,11 @@ public class TrashPolicyDefault extends TrashPolicy {
     private long floor(long time, long interval) {
       return (time / interval) * interval;
     }
+
+    @VisibleForTesting
+    protected long getEmptierInterval() {
+      return this.emptierInterval/MSECS_PER_MINUTE;
+    }
   }
 
   private void createCheckpoint(Path trashRoot, Date date) throws IOException {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java

@@ -183,6 +183,18 @@ public class FsPermission implements Writable {
     return toShort();
   }
 
+  /**
+   * Returns the FsPermission in an octal format.
+   *
+   * @return short Unlike {@link #toShort()} which provides a binary
+   * representation, this method returns the standard octal style permission.
+   */
+  public short toOctal() {
+    int n = this.toShort();
+    int octal = (n>>>9&1)*1000 + (n>>>6&7)*100 + (n>>>3&7)*10 + (n&7);
+    return (short)octal;
+  }
+
   @Override
   public boolean equals(Object obj) {
     if (obj instanceof FsPermission) {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java

@@ -31,6 +31,8 @@ import org.apache.hadoop.fs.FileStatus;
 /**
  * Print statistics about path in specified format.
  * Format sequences:<br>
+ *   %a: Permissions in octal<br>
+ *   %A: Permissions in symbolic style<br>
  *   %b: Size of file in blocks<br>
  *   %F: Type<br>
  *   %g: Group name of owner<br>
@@ -56,7 +58,8 @@ class Stat extends FsCommand {
   public static final String USAGE = "[format] <path> ...";
   public static final String DESCRIPTION =
     "Print statistics about the file/directory at <path>" + NEWLINE +
-    "in the specified format. Format accepts filesize in" + NEWLINE +
+    "in the specified format. Format accepts permissions in" + NEWLINE +
+    "octal (%a) and symbolic (%A), filesize in" + NEWLINE +
     "blocks (%b), type (%F), group name of owner (%g)," + NEWLINE +
     "name (%n), block size (%o), replication (%r), user name" + NEWLINE +
     "of owner (%u), modification date (%y, %Y)." + NEWLINE +
@@ -95,6 +98,12 @@ class Stat extends FsCommand {
         // this silently drops a trailing %?
         if (i + 1 == fmt.length) break;
         switch (fmt[++i]) {
+          case 'a':
+            buf.append(stat.getPermission().toOctal());
+            break;
+          case 'A':
+            buf.append(stat.getPermission());
+            break;
           case 'b':
             buf.append(stat.getLen());
             break;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java

@@ -354,6 +354,11 @@ class ChRootedFileSystem extends FilterFileSystem {
     return super.getXAttrs(fullPath(path), names);
   }
 
+  @Override
+  public boolean truncate(Path path, long newLength) throws IOException {
+    return super.truncate(fullPath(path), newLength);
+  }
+
   @Override
   public List<String> listXAttrs(Path path) throws IOException {
     return super.listXAttrs(fullPath(path));

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java

@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -36,47 +36,45 @@ import org.apache.hadoop.fs.UnsupportedFileSystemException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.StringUtils;
 
-
 /**
  * InodeTree implements a mount-table as a tree of inodes.
  * It is used to implement ViewFs and ViewFileSystem.
  * In order to use it the caller must subclass it and implement
  * the abstract methods {@link #getTargetFileSystem(INodeDir)}, etc.
- * 
+ *
  * The mountable is initialized from the config variables as 
  * specified in {@link ViewFs}
  *
  * @param <T> is AbstractFileSystem or FileSystem
- * 
- * The three main methods are
- * {@link #InodeTreel(Configuration)} // constructor
+ *
+ * The two main methods are
  * {@link #InodeTree(Configuration, String)} // constructor
  * {@link #resolve(String, boolean)} 
  */
 
 @InterfaceAudience.Private
-@InterfaceStability.Unstable 
+@InterfaceStability.Unstable
 abstract class InodeTree<T> {
-  static enum ResultKind {isInternalDir, isExternalDir;};
+  enum ResultKind {
+    INTERNAL_DIR,
+    EXTERNAL_DIR
+  }
+
   static final Path SlashPath = new Path("/");
-  
-  final INodeDir<T> root; // the root of the mount table
-  
-  final String homedirPrefix; // the homedir config value for this mount table
-  
-  List<MountPoint<T>> mountPoints = new ArrayList<MountPoint<T>>();
-  
-  
+  private final INodeDir<T> root;     // the root of the mount table
+  private final String homedirPrefix; // the homedir for this mount table
+  private List<MountPoint<T>> mountPoints = new ArrayList<MountPoint<T>>();
+
   static class MountPoint<T> {
     String src;
     INodeLink<T> target;
+
     MountPoint(String srcPath, INodeLink<T> mountLink) {
       src = srcPath;
       target = mountLink;
     }
-
   }
-  
+
   /**
    * Breaks file path into component names.
    * @param path
@@ -84,18 +82,19 @@ abstract class InodeTree<T> {
    */
   static String[] breakIntoPathComponents(final String path) {
     return path == null ? null : path.split(Path.SEPARATOR);
-  } 
-  
+  }
+
   /**
    * Internal class for inode tree
    * @param <T>
    */
   abstract static class INode<T> {
     final String fullPath; // the full path to the root
+
     public INode(String pathToNode, UserGroupInformation aUgi) {
       fullPath = pathToNode;
     }
-  };
+  }
 
   /**
    * Internal class to represent an internal dir of the mount table
@@ -105,37 +104,28 @@ abstract class InodeTree<T> {
     final Map<String,INode<T>> children = new HashMap<String,INode<T>>();
     T InodeDirFs =  null; // file system of this internal directory of mountT
     boolean isRoot = false;
-    
+
     INodeDir(final String pathToNode, final UserGroupInformation aUgi) {
       super(pathToNode, aUgi);
     }
 
-    INode<T> resolve(final String pathComponent) throws FileNotFoundException {
-      final INode<T> result = resolveInternal(pathComponent);
-      if (result == null) {
-        throw new FileNotFoundException();
-      }
-      return result;
-    }
-    
     INode<T> resolveInternal(final String pathComponent) {
       return children.get(pathComponent);
     }
-    
+
     INodeDir<T> addDir(final String pathComponent,
-        final UserGroupInformation aUgi)
-      throws FileAlreadyExistsException {
+        final UserGroupInformation aUgi) throws FileAlreadyExistsException {
       if (children.containsKey(pathComponent)) {
         throw new FileAlreadyExistsException();
       }
-      final INodeDir<T> newDir = new INodeDir<T>(fullPath+ (isRoot ? "" : "/") + 
-          pathComponent, aUgi);
+      final INodeDir<T> newDir = new INodeDir<T>(fullPath +
+          (isRoot ? "" : "/") + pathComponent, aUgi);
       children.put(pathComponent, newDir);
       return newDir;
     }
-    
+
     void addLink(final String pathComponent, final INodeLink<T> link)
-      throws FileAlreadyExistsException {
+        throws FileAlreadyExistsException {
       if (children.containsKey(pathComponent)) {
         throw new FileAlreadyExistsException();
       }
@@ -144,14 +134,14 @@ abstract class InodeTree<T> {
   }
 
   /**
-   * In internal class to represent a mount link
+   * An internal class to represent a mount link.
    * A mount link can be single dir link or a merge dir link.
 
    * A merge dir link is  a merge (junction) of links to dirs:
-   * example : <merge of 2 dirs
+   * example : merge of 2 dirs
    *     /users -> hdfs:nn1//users
    *     /users -> hdfs:nn2//users
-   * 
+   *
    * For a merge, each target is checked to be dir when created but if target
    * is changed later it is then ignored (a dir with null entries)
    */
@@ -159,9 +149,9 @@ abstract class InodeTree<T> {
     final boolean isMergeLink; // true if MergeLink
     final URI[] targetDirLinkList;
     final T targetFileSystem;   // file system object created from the link.
-    
+
     /**
-     * Construct a mergeLink
+     * Construct a mergeLink.
      */
     INodeLink(final String pathToNode, final UserGroupInformation aUgi,
         final T targetMergeFs, final URI[] aTargetDirLinkList) {
@@ -170,9 +160,9 @@ abstract class InodeTree<T> {
       targetDirLinkList = aTargetDirLinkList;
       isMergeLink = true;
     }
-    
+
     /**
-     * Construct a simple link (i.e. not a mergeLink)
+     * Construct a simple link (i.e. not a mergeLink).
      */
     INodeLink(final String pathToNode, final UserGroupInformation aUgi,
         final T targetFs, final URI aTargetDirLink) {
@@ -182,38 +172,36 @@ abstract class InodeTree<T> {
       targetDirLinkList[0] = aTargetDirLink;
       isMergeLink = false;
     }
-    
+
     /**
-     * Get the target of the link
-     * If a merge link then it returned as "," separated URI list.
+     * Get the target of the link. If a merge link then it returned
+     * as "," separated URI list.
      */
     Path getTargetLink() {
-      // is merge link - use "," as separator between the merged URIs
-      //String result = targetDirLinkList[0].toString();
       StringBuilder result = new StringBuilder(targetDirLinkList[0].toString());
-      for (int i=1; i < targetDirLinkList.length; ++i) { 
+      // If merge link, use "," as separator between the merged URIs
+      for (int i = 1; i < targetDirLinkList.length; ++i) {
         result.append(',').append(targetDirLinkList[i].toString());
       }
       return new Path(result.toString());
     }
   }
 
-
   private void createLink(final String src, final String target,
       final boolean isLinkMerge, final UserGroupInformation aUgi)
       throws URISyntaxException, IOException,
-    FileAlreadyExistsException, UnsupportedFileSystemException {
+      FileAlreadyExistsException, UnsupportedFileSystemException {
     // Validate that src is valid absolute path
-    final Path srcPath = new Path(src); 
+    final Path srcPath = new Path(src);
     if (!srcPath.isAbsoluteAndSchemeAuthorityNull()) {
-      throw new IOException("ViewFs:Non absolute mount name in config:" + src);
+      throw new IOException("ViewFs: Non absolute mount name in config:" + src);
     }
- 
+
     final String[] srcPaths = breakIntoPathComponents(src);
     INodeDir<T> curInode = root;
     int i;
     // Ignore first initial slash, process all except last component
-    for (i = 1; i < srcPaths.length-1; i++) {
+    for (i = 1; i < srcPaths.length - 1; i++) {
       final String iPath = srcPaths[i];
       INode<T> nextInode = curInode.resolveInternal(iPath);
       if (nextInode == null) {
@@ -226,11 +214,11 @@ abstract class InodeTree<T> {
         throw new FileAlreadyExistsException("Path " + nextInode.fullPath +
             " already exists as link");
       } else {
-        assert(nextInode instanceof INodeDir);
+        assert (nextInode instanceof INodeDir);
         curInode = (INodeDir<T>) nextInode;
       }
     }
-    
+
     // Now process the last component
     // Add the link in 2 cases: does not exist or a link exists
     String iPath = srcPaths[i];// last component
@@ -241,9 +229,9 @@ abstract class InodeTree<T> {
         strB.append('/').append(srcPaths[j]);
       }
       throw new FileAlreadyExistsException("Path " + strB +
-            " already exists as dir; cannot create link here");
+          " already exists as dir; cannot create link here");
     }
-    
+
     final INodeLink<T> newLink;
     final String fullPath = curInode.fullPath + (curInode == root ? "" : "/")
         + iPath;
@@ -263,25 +251,21 @@ abstract class InodeTree<T> {
     curInode.addLink(iPath, newLink);
     mountPoints.add(new MountPoint<T>(src, newLink));
   }
-  
-  /**
-   * Below the "public" methods of InodeTree
-   */
-  
+
   /**
    * The user of this class must subclass and implement the following
    * 3 abstract methods.
-   * @throws IOException 
+   * @throws IOException
    */
   protected abstract T getTargetFileSystem(final URI uri)
-    throws UnsupportedFileSystemException, URISyntaxException, IOException;
-  
+      throws UnsupportedFileSystemException, URISyntaxException, IOException;
+
   protected abstract T getTargetFileSystem(final INodeDir<T> dir)
-    throws URISyntaxException;
-  
+      throws URISyntaxException;
+
   protected abstract T getTargetFileSystem(final URI[] mergeFsURIList)
-  throws UnsupportedFileSystemException, URISyntaxException;
-  
+      throws UnsupportedFileSystemException, URISyntaxException;
+
   /**
    * Create Inode Tree from the specified mount-table specified in Config
    * @param config - the mount table keys are prefixed with 
@@ -294,7 +278,7 @@ abstract class InodeTree<T> {
    */
   protected InodeTree(final Configuration config, final String viewName)
       throws UnsupportedFileSystemException, URISyntaxException,
-    FileAlreadyExistsException, IOException { 
+      FileAlreadyExistsException, IOException {
     String vName = viewName;
     if (vName == null) {
       vName = Constants.CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE;
@@ -303,9 +287,9 @@ abstract class InodeTree<T> {
     root = new INodeDir<T>("/", UserGroupInformation.getCurrentUser());
     root.InodeDirFs = getTargetFileSystem(root);
     root.isRoot = true;
-    
-    final String mtPrefix = Constants.CONFIG_VIEWFS_PREFIX + "." + 
-                            vName + ".";
+
+    final String mtPrefix = Constants.CONFIG_VIEWFS_PREFIX + "." +
+        vName + ".";
     final String linkPrefix = Constants.CONFIG_VIEWFS_LINK + ".";
     final String linkMergePrefix = Constants.CONFIG_VIEWFS_LINK_MERGE + ".";
     boolean gotMountTableEntry = false;
@@ -325,18 +309,17 @@ abstract class InodeTree<T> {
           // ignore - we set home dir from config
           continue;
         } else {
-          throw new IOException(
-          "ViewFs: Cannot initialize: Invalid entry in Mount table in config: "+ 
-          src);
+          throw new IOException("ViewFs: Cannot initialize: Invalid entry in " +
+              "Mount table in config: " + src);
         }
         final String target = si.getValue(); // link or merge link
-        createLink(src, target, isMergeLink, ugi); 
+        createLink(src, target, isMergeLink, ugi);
       }
     }
     if (!gotMountTableEntry) {
       throw new IOException(
           "ViewFs: Cannot initialize: Empty Mount table in config for " +
-             "viewfs://" + vName + "/");
+              "viewfs://" + vName + "/");
     }
   }
 
@@ -344,7 +327,7 @@ abstract class InodeTree<T> {
    * Resolve returns ResolveResult.
    * The caller can continue the resolution of the remainingPath
    * in the targetFileSystem.
-   * 
+   *
    * If the input pathname leads to link to another file system then
    * the targetFileSystem is the one denoted by the link (except it is
    * file system chrooted to link target.
@@ -356,7 +339,7 @@ abstract class InodeTree<T> {
     final T targetFileSystem;
     final String resolvedPath;
     final Path remainingPath;   // to resolve in the target FileSystem
-    
+
     ResolveResult(final ResultKind k, final T targetFs, final String resolveP,
         final Path remainingP) {
       kind = k;
@@ -364,31 +347,30 @@ abstract class InodeTree<T> {
       resolvedPath = resolveP;
       remainingPath = remainingP;
     }
-    
-    // isInternalDir of path resolution completed within the mount table 
+
+    // Internal dir path resolution completed within the mount table
     boolean isInternalDir() {
-      return (kind == ResultKind.isInternalDir);
+      return (kind == ResultKind.INTERNAL_DIR);
     }
   }
-  
+
   /**
    * Resolve the pathname p relative to root InodeDir
    * @param p - inout path
-   * @param resolveLastComponent 
+   * @param resolveLastComponent
    * @return ResolveResult which allows further resolution of the remaining path
    * @throws FileNotFoundException
    */
   ResolveResult<T> resolve(final String p, final boolean resolveLastComponent)
-    throws FileNotFoundException {
-    // TO DO: - more efficient to not split the path, but simply compare
-    String[] path = breakIntoPathComponents(p); 
+      throws FileNotFoundException {
+    String[] path = breakIntoPathComponents(p);
     if (path.length <= 1) { // special case for when path is "/"
-      ResolveResult<T> res = 
-        new ResolveResult<T>(ResultKind.isInternalDir, 
+      ResolveResult<T> res =
+          new ResolveResult<T>(ResultKind.INTERNAL_DIR,
               root.InodeDirFs, root.fullPath, SlashPath);
       return res;
     }
-    
+
     INodeDir<T> curInode = root;
     int i;
     // ignore first slash
@@ -396,27 +378,27 @@ abstract class InodeTree<T> {
       INode<T> nextInode = curInode.resolveInternal(path[i]);
       if (nextInode == null) {
         StringBuilder failedAt = new StringBuilder(path[0]);
-        for ( int j = 1; j <=i; ++j) {
+        for (int j = 1; j <= i; ++j) {
           failedAt.append('/').append(path[j]);
         }
-        throw (new FileNotFoundException(failedAt.toString()));      
+        throw (new FileNotFoundException(failedAt.toString()));
       }
 
       if (nextInode instanceof INodeLink) {
         final INodeLink<T> link = (INodeLink<T>) nextInode;
         final Path remainingPath;
-        if (i >= path.length-1) {
+        if (i >= path.length - 1) {
           remainingPath = SlashPath;
         } else {
-          StringBuilder remainingPathStr = new StringBuilder("/" + path[i+1]);
-          for (int j = i+2; j< path.length; ++j) {
+          StringBuilder remainingPathStr = new StringBuilder("/" + path[i + 1]);
+          for (int j = i + 2; j < path.length; ++j) {
             remainingPathStr.append('/').append(path[j]);
           }
           remainingPath = new Path(remainingPathStr.toString());
         }
-        final ResolveResult<T> res = 
-          new ResolveResult<T>(ResultKind.isExternalDir,
-              link.targetFileSystem, nextInode.fullPath, remainingPath);
+        final ResolveResult<T> res =
+            new ResolveResult<T>(ResultKind.EXTERNAL_DIR,
+                link.targetFileSystem, nextInode.fullPath, remainingPath);
         return res;
       } else if (nextInode instanceof INodeDir) {
         curInode = (INodeDir<T>) nextInode;
@@ -433,23 +415,23 @@ abstract class InodeTree<T> {
       // that follows will do a children.get(remaningPath) and will have to
       // strip-out the initial /
       StringBuilder remainingPathStr = new StringBuilder("/" + path[i]);
-      for (int j = i+1; j< path.length; ++j) {
+      for (int j = i + 1; j < path.length; ++j) {
         remainingPathStr.append('/').append(path[j]);
       }
       remainingPath = new Path(remainingPathStr.toString());
     }
-    final ResolveResult<T> res = 
-       new ResolveResult<T>(ResultKind.isInternalDir,
-           curInode.InodeDirFs, curInode.fullPath, remainingPath); 
+    final ResolveResult<T> res =
+        new ResolveResult<T>(ResultKind.INTERNAL_DIR,
+            curInode.InodeDirFs, curInode.fullPath, remainingPath);
     return res;
   }
-  
-  List<MountPoint<T>> getMountPoints() { 
+
+  List<MountPoint<T>> getMountPoints() {
     return mountPoints;
   }
-  
+
   /**
-   * 
+   *
    * @return home dir value from mount table; null if no config value
    * was found.
    */

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java

@@ -87,10 +87,19 @@ public class ViewFileSystem extends FileSystem {
       final Path p) {
     return readOnlyMountTable(operation, p.toString());
   }
-  
+
   static public class MountPoint {
-    private Path src;       // the src of the mount
-    private URI[] targets; //  target of the mount; Multiple targets imply mergeMount
+    /**
+     *  The source of the mount.
+     */
+    private Path src;
+
+    /**
+     * One or more targets of the mount.
+     * Multiple targets imply MergeMount.
+     */
+    private URI[] targets;
+
     MountPoint(Path srcPath, URI[] targetURIs) {
       src = srcPath;
       targets = targetURIs;
@@ -142,19 +151,18 @@ public class ViewFileSystem extends FileSystem {
 
   /**
    * Return the protocol scheme for the FileSystem.
-   * <p/>
    *
    * @return <code>viewfs</code>
    */
   @Override
   public String getScheme() {
-    return "viewfs";
+    return FsConstants.VIEWFS_SCHEME;
   }
 
   /**
    * Called after a new FileSystem instance is constructed.
    * @param theUri a uri whose authority section names the host, port, etc. for
-   *          this FileSystem
+   *        this FileSystem
    * @param conf the configuration
    */
   @Override
@@ -197,8 +205,7 @@ public class ViewFileSystem extends FileSystem {
     }
 
   }
-  
-  
+
   /**
    * Convenience Constructor for apps to call directly
    * @param theUri which must be that of ViewFileSystem
@@ -206,7 +213,7 @@ public class ViewFileSystem extends FileSystem {
    * @throws IOException
    */
   ViewFileSystem(final URI theUri, final Configuration conf)
-    throws IOException {
+      throws IOException {
     this();
     initialize(theUri, conf);
   }
@@ -220,20 +227,13 @@ public class ViewFileSystem extends FileSystem {
     this(FsConstants.VIEWFS_URI, conf);
   }
   
-  public Path getTrashCanLocation(final Path f) throws FileNotFoundException {
-    final InodeTree.ResolveResult<FileSystem> res = 
-      fsState.resolve(getUriPath(f), true);
-    return res.isInternalDir() ? null : res.targetFileSystem.getHomeDirectory();
-  }
-  
   @Override
   public URI getUri() {
     return myUri;
   }
   
   @Override
-  public Path resolvePath(final Path f)
-      throws IOException {
+  public Path resolvePath(final Path f) throws IOException {
     final InodeTree.ResolveResult<FileSystem> res;
       res = fsState.resolve(getUriPath(f), true);
     if (res.isInternalDir()) {
@@ -277,8 +277,8 @@ public class ViewFileSystem extends FileSystem {
   
   @Override
   public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
-      EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
-      Progressable progress) throws IOException {
+      EnumSet<CreateFlag> flags, int bufferSize, short replication,
+      long blockSize, Progressable progress) throws IOException {
     InodeTree.ResolveResult<FileSystem> res;
     try {
       res = fsState.resolve(getUriPath(f), false);
@@ -286,8 +286,8 @@ public class ViewFileSystem extends FileSystem {
         throw readOnlyMountTable("create", f);
     }
     assert(res.remainingPath != null);
-    return res.targetFileSystem.createNonRecursive(res.remainingPath, permission,
-        flags, bufferSize, replication, blockSize, progress);
+    return res.targetFileSystem.createNonRecursive(res.remainingPath,
+        permission, flags, bufferSize, replication, blockSize, progress);
   }
   
   @Override
@@ -308,10 +308,9 @@ public class ViewFileSystem extends FileSystem {
   
   @Override
   public boolean delete(final Path f, final boolean recursive)
-      throws AccessControlException, FileNotFoundException,
-      IOException {
+      throws AccessControlException, FileNotFoundException, IOException {
     InodeTree.ResolveResult<FileSystem> res = 
-      fsState.resolve(getUriPath(f), true);
+        fsState.resolve(getUriPath(f), true);
     // If internal dir or target is a mount link (ie remainingPath is Slash)
     if (res.isInternalDir() || res.remainingPath == InodeTree.SlashPath) {
       throw readOnlyMountTable("delete", f);
@@ -322,9 +321,8 @@ public class ViewFileSystem extends FileSystem {
   @Override
   @SuppressWarnings("deprecation")
   public boolean delete(final Path f)
-      throws AccessControlException, FileNotFoundException,
-      IOException {
-      return delete(f, true);
+      throws AccessControlException, FileNotFoundException, IOException {
+    return delete(f, true);
   }
   
   @Override
@@ -345,7 +343,6 @@ public class ViewFileSystem extends FileSystem {
     return res.targetFileSystem.getFileChecksum(res.remainingPath);
   }
 
-
   private static FileStatus fixFileStatus(FileStatus orig,
       Path qualified) throws IOException {
     // FileStatus#getPath is a fully qualified path relative to the root of
@@ -373,7 +370,6 @@ public class ViewFileSystem extends FileSystem {
         : new ViewFsFileStatus(orig, qualified);
   }
 
-
   @Override
   public FileStatus getFileStatus(final Path f) throws AccessControlException,
       FileNotFoundException, IOException {
@@ -413,10 +409,10 @@ public class ViewFileSystem extends FileSystem {
   @Override
   public RemoteIterator<LocatedFileStatus>listLocatedStatus(final Path f,
       final PathFilter filter) throws FileNotFoundException, IOException {
-    final InodeTree.ResolveResult<FileSystem> res = fsState
-        .resolve(getUriPath(f), true);
-    final RemoteIterator<LocatedFileStatus> statusIter = res.targetFileSystem
-        .listLocatedStatus(res.remainingPath);
+    final InodeTree.ResolveResult<FileSystem> res =
+        fsState.resolve(getUriPath(f), true);
+    final RemoteIterator<LocatedFileStatus> statusIter =
+        res.targetFileSystem.listLocatedStatus(res.remainingPath);
 
     if (res.isInternalDir()) {
       return statusIter;
@@ -455,8 +451,7 @@ public class ViewFileSystem extends FileSystem {
 
   @Override
   public FSDataInputStream open(final Path f, final int bufferSize)
-      throws AccessControlException, FileNotFoundException,
-      IOException {
+      throws AccessControlException, FileNotFoundException, IOException {
     InodeTree.ResolveResult<FileSystem> res = 
         fsState.resolve(getUriPath(f), true);
     return res.targetFileSystem.open(res.remainingPath, bufferSize);
@@ -507,14 +502,13 @@ public class ViewFileSystem extends FileSystem {
       throws IOException {
     InodeTree.ResolveResult<FileSystem> res =
         fsState.resolve(getUriPath(f), true);
-    return res.targetFileSystem.truncate(f, newLength);
+    return res.targetFileSystem.truncate(res.remainingPath, newLength);
   }
   
   @Override
   public void setOwner(final Path f, final String username,
       final String groupname) throws AccessControlException,
-      FileNotFoundException,
-      IOException {
+      FileNotFoundException, IOException {
     InodeTree.ResolveResult<FileSystem> res = 
       fsState.resolve(getUriPath(f), true);
     res.targetFileSystem.setOwner(res.remainingPath, username, groupname); 
@@ -522,8 +516,7 @@ public class ViewFileSystem extends FileSystem {
 
   @Override
   public void setPermission(final Path f, final FsPermission permission)
-      throws AccessControlException, FileNotFoundException,
-      IOException {
+      throws AccessControlException, FileNotFoundException, IOException {
     InodeTree.ResolveResult<FileSystem> res = 
       fsState.resolve(getUriPath(f), true);
     res.targetFileSystem.setPermission(res.remainingPath, permission); 
@@ -531,8 +524,7 @@ public class ViewFileSystem extends FileSystem {
 
   @Override
   public boolean setReplication(final Path f, final short replication)
-      throws AccessControlException, FileNotFoundException,
-      IOException {
+      throws AccessControlException, FileNotFoundException, IOException {
     InodeTree.ResolveResult<FileSystem> res = 
       fsState.resolve(getUriPath(f), true);
     return res.targetFileSystem.setReplication(res.remainingPath, replication);
@@ -540,8 +532,7 @@ public class ViewFileSystem extends FileSystem {
 
   @Override
   public void setTimes(final Path f, final long mtime, final long atime)
-      throws AccessControlException, FileNotFoundException,
-      IOException {
+      throws AccessControlException, FileNotFoundException, IOException {
     InodeTree.ResolveResult<FileSystem> res = 
       fsState.resolve(getUriPath(f), true);
     res.targetFileSystem.setTimes(res.remainingPath, mtime, atime); 
@@ -799,8 +790,8 @@ public class ViewFileSystem extends FileSystem {
     return allPolicies;
   }
 
-  /*
-   * An instance of this class represents an internal dir of the viewFs 
+  /**
+   * An instance of this class represents an internal dir of the viewFs
    * that is internal dir of the mount table.
    * It is a read only mount tables and create, mkdir or delete operations
    * are not allowed.
@@ -832,8 +823,8 @@ public class ViewFileSystem extends FileSystem {
 
     static private void checkPathIsSlash(final Path f) throws IOException {
       if (f != InodeTree.SlashPath) {
-        throw new IOException (
-        "Internal implementation error: expected file name to be /" );
+        throw new IOException(
+            "Internal implementation error: expected file name to be /");
       }
     }
     
@@ -844,14 +835,14 @@ public class ViewFileSystem extends FileSystem {
 
     @Override
     public Path getWorkingDirectory() {
-      throw new RuntimeException (
-      "Internal impl error: getWorkingDir should not have been called" );
+      throw new RuntimeException(
+          "Internal impl error: getWorkingDir should not have been called");
     }
 
     @Override
     public void setWorkingDirectory(final Path new_dir) {
-      throw new RuntimeException (
-      "Internal impl error: getWorkingDir should not have been called" ); 
+      throw new RuntimeException(
+          "Internal impl error: getWorkingDir should not have been called");
     }
 
     @Override
@@ -884,7 +875,7 @@ public class ViewFileSystem extends FileSystem {
 
     @Override
     public BlockLocation[] getFileBlockLocations(final FileStatus fs,
-        final long start, final long len) throws 
+        final long start, final long len) throws
         FileNotFoundException, IOException {
       checkPathIsSlash(fs.getPath());
       throw new FileNotFoundException("Path points to dir not a file");
@@ -1061,7 +1052,7 @@ public class ViewFileSystem extends FileSystem {
 
     @Override
     public void setXAttr(Path path, String name, byte[] value,
-                         EnumSet<XAttrSetFlag> flag) throws IOException {
+        EnumSet<XAttrSetFlag> flag) throws IOException {
       checkPathIsSlash(path);
       throw readOnlyMountTable("setXAttr", path);
     }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java

@@ -20,19 +20,107 @@ package org.apache.hadoop.io.erasurecode;
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.erasurecode.codec.ErasureCodec;
+import org.apache.hadoop.io.erasurecode.codec.HHXORErasureCodec;
+import org.apache.hadoop.io.erasurecode.codec.RSErasureCodec;
+import org.apache.hadoop.io.erasurecode.codec.XORErasureCodec;
+import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder;
+import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 
 /**
- * A codec & coder utility to help create raw coders conveniently.
+ * A codec & coder utility to help create coders conveniently.
+ *
+ * {@link CodecUtil} includes erasure coder configurations key and default
+ * values such as coder class name and erasure codec option values included
+ * by {@link ErasureCodecOptions}. {@link ErasureEncoder} and
+ * {@link ErasureDecoder} are created by createEncoder and createDecoder
+ * respectively.{@link RawErasureEncoder} and {@link RawErasureDecoder} are
+ * are created by createRawEncoder and createRawDecoder.
  */
 @InterfaceAudience.Private
 public final class CodecUtil {
 
+  /** Erasure coder XOR codec. */
+  public static final String IO_ERASURECODE_CODEC_XOR_KEY =
+      "io.erasurecode.codec.xor";
+  public static final String IO_ERASURECODE_CODEC_XOR =
+      XORErasureCodec.class.getCanonicalName();
+  /** Erasure coder Reed-Solomon codec. */
+  public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_KEY =
+      "io.erasurecode.codec.rs";
+  public static final String IO_ERASURECODE_CODEC_RS_DEFAULT =
+      RSErasureCodec.class.getCanonicalName();
+  /** Erasure coder hitch hiker XOR codec. */
+  public static final String IO_ERASURECODE_CODEC_HHXOR_KEY =
+      "io.erasurecode.codec.hhxor";
+  public static final String IO_ERASURECODE_CODEC_HHXOR =
+      HHXORErasureCodec.class.getCanonicalName();
+
+  /** Supported erasure codec classes. */
+
+  /** Raw coder factory for the RS default codec. */
+  public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY =
+      "io.erasurecode.codec.rs-default.rawcoder";
+  public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT =
+      RSRawErasureCoderFactory.class.getCanonicalName();
+
+  /** Raw coder factory for the RS legacy codec. */
+  public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY =
+      "io.erasurecode.codec.rs-legacy.rawcoder";
+  public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT =
+      RSRawErasureCoderFactoryLegacy.class.getCanonicalName();
+
+  /** Raw coder factory for the XOR codec. */
+  public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY =
+      "io.erasurecode.codec.xor.rawcoder";
+  public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT =
+      XORRawErasureCoderFactory.class.getCanonicalName();
+
   private CodecUtil() { }
 
+  /**
+   * Create encoder corresponding to given codec.
+   * @param options Erasure codec options
+   * @return erasure encoder
+   */
+  public static ErasureEncoder createEncoder(Configuration conf,
+      ErasureCodecOptions options) {
+    Preconditions.checkNotNull(conf);
+    Preconditions.checkNotNull(options);
+
+    String codecKey = getCodecClassName(conf,
+        options.getSchema().getCodecName());
+
+    ErasureCodec codec = createCodec(conf, codecKey, options);
+    return codec.createEncoder();
+  }
+
+  /**
+   * Create decoder corresponding to given codec.
+   * @param options Erasure codec options
+   * @return erasure decoder
+   */
+  public static ErasureDecoder createDecoder(Configuration conf,
+      ErasureCodecOptions options) {
+    Preconditions.checkNotNull(conf);
+    Preconditions.checkNotNull(options);
+
+    String codecKey = getCodecClassName(conf,
+        options.getSchema().getCodecName());
+
+    ErasureCodec codec = createCodec(conf, codecKey, options);
+    return codec.createDecoder();
+  }
+
   /**
    * Create RS raw encoder according to configuration.
    * @param conf configuration
@@ -45,7 +133,7 @@ public final class CodecUtil {
     Preconditions.checkNotNull(conf);
     Preconditions.checkNotNull(codec);
 
-    String rawCoderFactoryKey = getFactNameFromCodec(conf, codec);
+    String rawCoderFactoryKey = getRawCoderFactNameFromCodec(conf, codec);
 
     RawErasureCoderFactory fact = createRawCoderFactory(conf,
         rawCoderFactoryKey);
@@ -65,7 +153,7 @@ public final class CodecUtil {
     Preconditions.checkNotNull(conf);
     Preconditions.checkNotNull(codec);
 
-    String rawCoderFactoryKey = getFactNameFromCodec(conf, codec);
+    String rawCoderFactoryKey = getRawCoderFactNameFromCodec(conf, codec);
 
     RawErasureCoderFactory fact = createRawCoderFactory(conf,
         rawCoderFactoryKey);
@@ -92,22 +180,21 @@ public final class CodecUtil {
     return fact;
   }
 
-  private static String getFactNameFromCodec(Configuration conf, String codec) {
+  private static String getRawCoderFactNameFromCodec(Configuration conf,
+                                                     String codec) {
     switch (codec) {
     case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME:
       return conf.get(
-          CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
-          CommonConfigurationKeys.
-              IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT);
+          IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+          IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT);
     case ErasureCodeConstants.RS_LEGACY_CODEC_NAME:
       return conf.get(
-          CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY,
-          CommonConfigurationKeys.
-              IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT);
+          IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY,
+          IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT);
     case ErasureCodeConstants.XOR_CODEC_NAME:
       return conf.get(
-          CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY,
-          CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT);
+          IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY,
+          IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT);
     default:
       // For custom codec, we throw exception if the factory is not configured
       String rawCoderKey = "io.erasurecode.codec." + codec + ".rawcoder";
@@ -119,4 +206,59 @@ public final class CodecUtil {
       return factName;
     }
   }
+
+  private static ErasureCodec createCodec(Configuration conf,
+      String codecClassName, ErasureCodecOptions options) {
+    ErasureCodec codec = null;
+    try {
+      Class<? extends ErasureCodec> codecClass =
+              conf.getClassByName(codecClassName)
+              .asSubclass(ErasureCodec.class);
+      Constructor<? extends ErasureCodec> constructor
+          = codecClass.getConstructor(Configuration.class,
+          ErasureCodecOptions.class);
+      codec = constructor.newInstance(conf, options);
+    } catch (ClassNotFoundException | InstantiationException |
+            IllegalAccessException | NoSuchMethodException |
+            InvocationTargetException e) {
+      throw new RuntimeException("Failed to create erasure codec", e);
+    }
+
+    if (codec == null) {
+      throw new RuntimeException("Failed to create erasure codec");
+    }
+
+    return codec;
+  }
+
+  private static String getCodecClassName(Configuration conf, String codec) {
+    switch (codec) {
+    case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME:
+      return conf.get(
+          CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_KEY,
+          CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT);
+    case ErasureCodeConstants.RS_LEGACY_CODEC_NAME:
+      //TODO:rs-legacy should be handled differently.
+      return conf.get(
+          CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_KEY,
+          CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT);
+    case ErasureCodeConstants.XOR_CODEC_NAME:
+      return conf.get(
+          CodecUtil.IO_ERASURECODE_CODEC_XOR_KEY,
+          CodecUtil.IO_ERASURECODE_CODEC_XOR);
+    case ErasureCodeConstants.HHXOR_CODEC_NAME:
+      return conf.get(
+          CodecUtil.IO_ERASURECODE_CODEC_HHXOR_KEY,
+          CodecUtil.IO_ERASURECODE_CODEC_HHXOR);
+    default:
+      // For custom codec, we throw exception if the factory is not configured
+      String codecKey = "io.erasurecode.codec." + codec + ".coder";
+      String codecClass = conf.get(codecKey);
+      if (codecClass == null) {
+        throw new IllegalArgumentException("Codec not configured " +
+                "for custom codec " + codec);
+      }
+      return codecClass;
+    }
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java

@@ -22,12 +22,13 @@ package org.apache.hadoop.io.erasurecode;
  */
 public final class ErasureCodeConstants {
 
-  private ErasureCodeConstants(){
+  private ErasureCodeConstants() {
   }
 
   public static final String RS_DEFAULT_CODEC_NAME = "rs-default";
   public static final String RS_LEGACY_CODEC_NAME = "rs-legacy";
   public static final String XOR_CODEC_NAME = "xor";
+  public static final String HHXOR_CODEC_NAME = "hhxor";
 
   public static final ECSchema RS_6_3_SCHEMA = new ECSchema(
       RS_DEFAULT_CODEC_NAME, 6, 3);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java → hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodecOptions.java

@@ -15,39 +15,23 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.io.erasurecode.codec;
+
+package org.apache.hadoop.io.erasurecode;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.io.erasurecode.ECSchema;
-import org.apache.hadoop.io.erasurecode.grouper.BlockGrouper;
 
 /**
- * Abstract Erasure Codec that implements {@link ErasureCodec}.
+ * Erasure codec options.
  */
 @InterfaceAudience.Private
-public abstract class AbstractErasureCodec extends Configured
-    implements ErasureCodec {
-
-  private final ECSchema schema;
+public class ErasureCodecOptions {
+  private ECSchema schema;
 
-  public AbstractErasureCodec(ECSchema schema) {
+  public ErasureCodecOptions(ECSchema schema) {
     this.schema = schema;
   }
 
-  public String getName() {
-    return schema.getCodecName();
-  }
-
   public ECSchema getSchema() {
     return schema;
   }
-
-  @Override
-  public BlockGrouper createBlockGrouper() {
-    BlockGrouper blockGrouper = new BlockGrouper();
-    blockGrouper.setSchema(getSchema());
-
-    return blockGrouper;
-  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/DummyErasureCodec.java

@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.codec;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.erasurecode.ErasureCodecOptions;
+import org.apache.hadoop.io.erasurecode.coder.DummyErasureDecoder;
+import org.apache.hadoop.io.erasurecode.coder.DummyErasureEncoder;
+import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder;
+import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
+
+/**
+ * Dummy erasure coder does not real coding computing. This is used for only
+ * test or performance comparison with other erasure coders.
+ */
+public class DummyErasureCodec extends ErasureCodec {
+  public DummyErasureCodec(Configuration conf, ErasureCodecOptions options) {
+    super(conf, options);
+  }
+
+  @Override
+  public ErasureEncoder createEncoder() {
+    return new DummyErasureEncoder(getCoderOptions());
+  }
+
+  @Override
+  public ErasureDecoder createDecoder() {
+    return new DummyErasureDecoder(getCoderOptions());
+  }
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java

@@ -18,34 +18,76 @@
 package org.apache.hadoop.io.erasurecode.codec;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.io.erasurecode.coder.ErasureCoder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.ErasureCodecOptions;
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder;
+import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
 import org.apache.hadoop.io.erasurecode.grouper.BlockGrouper;
 
 /**
- * Erasure Codec API that's to cover the essential specific aspects of a code.
- * Currently it cares only block grouper and erasure coder. In future we may
- * add more aspects here to make the behaviors customizable.
+ * Abstract Erasure Codec is defines the interface of each actual erasure
+ * codec classes.
  */
 @InterfaceAudience.Private
-public interface ErasureCodec extends Configurable {
+public abstract class ErasureCodec {
 
-  /**
-   * Create block grouper
-   * @return block grouper
-   */
-  public BlockGrouper createBlockGrouper();
+  private ECSchema schema;
+  private ErasureCodecOptions codecOptions;
+  private ErasureCoderOptions coderOptions;
+
+  public ErasureCodec(Configuration conf,
+                      ErasureCodecOptions options) {
+    this.schema = options.getSchema();
+    this.codecOptions = options;
+    boolean allowChangeInputs = false;
+    this.coderOptions = new ErasureCoderOptions(schema.getNumDataUnits(),
+        schema.getNumParityUnits(), allowChangeInputs, false);
+  }
+
+  public String getName() {
+    return schema.getCodecName();
+  }
+
+  public ECSchema getSchema() {
+    return schema;
+  }
 
   /**
-   * Create Erasure Encoder
-   * @return erasure encoder
+   * Get a {@link ErasureCodecOptions}.
+   * @return erasure codec options
    */
-  public ErasureCoder createEncoder();
+  public ErasureCodecOptions getCodecOptions() {
+    return codecOptions;
+  }
+
+  protected void setCodecOptions(ErasureCodecOptions options) {
+    this.codecOptions = options;
+    this.schema = options.getSchema();
+  }
 
   /**
-   * Create Erasure Decoder
-   * @return erasure decoder
+   * Get a {@link ErasureCoderOptions}.
+   * @return erasure coder options
    */
-  public ErasureCoder createDecoder();
+  public ErasureCoderOptions getCoderOptions() {
+    return coderOptions;
+  }
+
+  protected void setCoderOptions(ErasureCoderOptions options) {
+    this.coderOptions = options;
+  }
+
+  public abstract ErasureEncoder createEncoder();
+
+  public abstract ErasureDecoder createDecoder();
+
+  public BlockGrouper createBlockGrouper() {
+    BlockGrouper blockGrouper = new BlockGrouper();
+    blockGrouper.setSchema(getSchema());
 
+    return blockGrouper;
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/HHXORErasureCodec.java

@@ -18,8 +18,10 @@
 package org.apache.hadoop.io.erasurecode.codec;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.erasurecode.ECSchema;
-import org.apache.hadoop.io.erasurecode.coder.ErasureCoder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.erasurecode.ErasureCodecOptions;
+import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder;
+import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
 import org.apache.hadoop.io.erasurecode.coder.HHXORErasureDecoder;
 import org.apache.hadoop.io.erasurecode.coder.HHXORErasureEncoder;
 
@@ -27,19 +29,19 @@ import org.apache.hadoop.io.erasurecode.coder.HHXORErasureEncoder;
  * A Hitchhiker-XOR erasure codec.
  */
 @InterfaceAudience.Private
-public class HHXORErasureCodec extends AbstractErasureCodec {
+public class HHXORErasureCodec extends ErasureCodec {
 
-  public HHXORErasureCodec(ECSchema schema) {
-    super(schema);
+  public HHXORErasureCodec(Configuration conf, ErasureCodecOptions options) {
+    super(conf, options);
   }
 
   @Override
-  public ErasureCoder createEncoder() {
-    return new HHXORErasureEncoder(getSchema());
+  public ErasureEncoder createEncoder() {
+    return new HHXORErasureEncoder(getCoderOptions());
   }
 
   @Override
-  public ErasureCoder createDecoder() {
-    return new HHXORErasureDecoder(getSchema());
+  public ErasureDecoder createDecoder() {
+    return new HHXORErasureDecoder(getCoderOptions());
   }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java

@@ -18,8 +18,10 @@
 package org.apache.hadoop.io.erasurecode.codec;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.erasurecode.ECSchema;
-import org.apache.hadoop.io.erasurecode.coder.ErasureCoder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.erasurecode.ErasureCodecOptions;
+import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder;
+import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
 import org.apache.hadoop.io.erasurecode.coder.RSErasureDecoder;
 import org.apache.hadoop.io.erasurecode.coder.RSErasureEncoder;
 
@@ -27,19 +29,19 @@ import org.apache.hadoop.io.erasurecode.coder.RSErasureEncoder;
  * A Reed-Solomon erasure codec.
  */
 @InterfaceAudience.Private
-public class RSErasureCodec extends AbstractErasureCodec {
+public class RSErasureCodec extends ErasureCodec {
 
-  public RSErasureCodec(ECSchema schema) {
-    super(schema);
+  public RSErasureCodec(Configuration conf, ErasureCodecOptions options) {
+    super(conf, options);
   }
 
   @Override
-  public ErasureCoder createEncoder() {
-    return new RSErasureEncoder(getSchema());
+  public ErasureEncoder createEncoder() {
+    return new RSErasureEncoder(getCoderOptions());
   }
 
   @Override
-  public ErasureCoder createDecoder() {
-    return new RSErasureDecoder(getSchema());
+  public ErasureDecoder createDecoder() {
+    return new RSErasureDecoder(getCoderOptions());
   }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java

@@ -18,8 +18,10 @@
 package org.apache.hadoop.io.erasurecode.codec;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.erasurecode.ECSchema;
-import org.apache.hadoop.io.erasurecode.coder.ErasureCoder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.erasurecode.ErasureCodecOptions;
+import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder;
+import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
 import org.apache.hadoop.io.erasurecode.coder.XORErasureDecoder;
 import org.apache.hadoop.io.erasurecode.coder.XORErasureEncoder;
 
@@ -27,20 +29,20 @@ import org.apache.hadoop.io.erasurecode.coder.XORErasureEncoder;
  * A XOR erasure codec.
  */
 @InterfaceAudience.Private
-public class XORErasureCodec extends AbstractErasureCodec {
+public class XORErasureCodec extends ErasureCodec {
 
-  public XORErasureCodec(ECSchema schema) {
-    super(schema);
-    assert(schema.getNumParityUnits() == 1);
+  public XORErasureCodec(Configuration conf, ErasureCodecOptions options) {
+    super(conf, options);
+    assert(options.getSchema().getNumParityUnits() == 1);
   }
 
   @Override
-  public ErasureCoder createEncoder() {
-    return new XORErasureEncoder(getSchema());
+  public ErasureEncoder createEncoder() {
+    return new XORErasureEncoder(getCoderOptions());
   }
 
   @Override
-  public ErasureCoder createDecoder() {
-    return new XORErasureDecoder(getSchema());
+  public ErasureDecoder createDecoder() {
+    return new XORErasureDecoder(getCoderOptions());
   }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/package-info.java

@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Erasure codec framework.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.io.erasurecode.codec;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java

@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.io.erasurecode.coder;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.io.erasurecode.ECSchema;
-
-/**
- * A common class of basic facilities to be shared by encoder and decoder
- *
- * It implements the {@link ErasureCoder} interface.
- */
-@InterfaceAudience.Private
-public abstract class AbstractErasureCoder
-    extends Configured implements ErasureCoder {
-
-  private final int numDataUnits;
-  private final int numParityUnits;
-
-  public AbstractErasureCoder(int numDataUnits, int numParityUnits) {
-    this.numDataUnits = numDataUnits;
-    this.numParityUnits = numParityUnits;
-  }
-
-  public AbstractErasureCoder(ECSchema schema) {
-    this(schema.getNumDataUnits(), schema.getNumParityUnits());
-  }
-
-  @Override
-  public int getNumDataUnits() {
-    return numDataUnits;
-  }
-
-  @Override
-  public int getNumParityUnits() {
-    return numParityUnits;
-  }
-
-  @Override
-  public boolean preferDirectBuffer() {
-    return false;
-  }
-
-  @Override
-  public void release() {
-    // Nothing to do by default
-  }
-}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/DummyErasureDecoder.java

@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import org.apache.hadoop.io.erasurecode.rawcoder.DummyRawDecoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+
+/**
+ * Dummy erasure decoder does no real computation. Instead, it just returns
+ * zero bytes. This decoder can be used to isolate the performance issue to
+ * HDFS side logic instead of codec, and is intended for test only.
+ */
+public class DummyErasureDecoder extends ErasureDecoder {
+  public DummyErasureDecoder(ErasureCoderOptions options) {
+    super(options);
+  }
+
+  @Override
+  protected ErasureCodingStep prepareDecodingStep(ECBlockGroup blockGroup) {
+    RawErasureDecoder rawDecoder = new DummyRawDecoder(getOptions());
+
+    ECBlock[] inputBlocks = getInputBlocks(blockGroup);
+
+    return new ErasureDecodingStep(inputBlocks,
+        getErasedIndexes(inputBlocks),
+        getOutputBlocks(blockGroup), rawDecoder);
+  }
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/DummyErasureEncoder.java

@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import org.apache.hadoop.io.erasurecode.rawcoder.DummyRawEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+
+/**
+ * Dummy erasure encoder does no real computation. Instead, it just returns
+ * zero bytes. This decoder can be used to isolate the performance issue to
+ * HDFS side logic instead of codec, and is intended for test only.
+ */
+public class DummyErasureEncoder extends ErasureEncoder {
+  public DummyErasureEncoder(ErasureCoderOptions options) {
+    super(options);
+  }
+
+  @Override
+  protected ErasureCodingStep prepareEncodingStep(ECBlockGroup blockGroup) {
+    RawErasureEncoder rawEncoder = new DummyRawEncoder(getOptions());
+
+    ECBlock[] inputBlocks = getInputBlocks(blockGroup);
+
+    return new ErasureEncodingStep(inputBlocks,
+        getOutputBlocks(blockGroup), rawEncoder);
+  }
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.io.erasurecode.coder;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 
 /**
  * An erasure coder to perform encoding or decoding given a group. Generally it
@@ -39,18 +40,25 @@ import org.apache.hadoop.io.erasurecode.ECBlockGroup;
 public interface ErasureCoder extends Configurable {
 
   /**
-   * The number of data input units for the coding. A unit can be a byte,
-   * chunk or buffer or even a block.
+   * The number of data input units for the coding. A unit can be a byte, chunk
+   * or buffer or even a block.
    * @return count of data input units
    */
-  public int getNumDataUnits();
+  int getNumDataUnits();
 
   /**
    * The number of parity output units for the coding. A unit can be a byte,
    * chunk, buffer or even a block.
    * @return count of parity output units
    */
-  public int getNumParityUnits();
+  int getNumParityUnits();
+
+  /**
+   * The options of erasure coder. This option is passed to
+   * raw erasure coder as it is.
+   * @return erasure coder options
+   */
+  ErasureCoderOptions getOptions();
 
   /**
    * Calculate the encoding or decoding steps given a block blockGroup.
@@ -61,7 +69,7 @@ public interface ErasureCoder extends Configurable {
    * @param blockGroup the erasure coding block group containing all necessary
    *                   information for codec calculation
    */
-  public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup);
+  ErasureCodingStep calculateCoding(ECBlockGroup blockGroup);
 
   /**
    * Tell if direct or off-heap buffer is preferred or not. It's for callers to
@@ -70,10 +78,11 @@ public interface ErasureCoder extends Configurable {
    * @return true if direct buffer is preferred for performance consideration,
    * otherwise false.
    */
-  public boolean preferDirectBuffer();
+  boolean preferDirectBuffer();
 
   /**
-   * Release the resources if any. Good chance to invoke RawErasureCoder#release.
+   * Release the resources if any. Good chance to invoke
+   * RawErasureCoder#release.
    */
-  public void release();
+  void release();
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java

@@ -32,14 +32,14 @@ public interface ErasureCodingStep {
    * or parity blocks.
    * @return input blocks
    */
-  public ECBlock[] getInputBlocks();
+  ECBlock[] getInputBlocks();
 
   /**
    * Output blocks of writable buffers involved in this step, may be data
    * blocks or parity blocks.
    * @return output blocks
    */
-  public ECBlock[] getOutputBlocks();
+  ECBlock[] getOutputBlocks();
 
   /**
    * Perform encoding or decoding given the input chunks, and generated results
@@ -47,11 +47,11 @@ public interface ErasureCodingStep {
    * @param inputChunks
    * @param outputChunks
    */
-  public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks);
+  void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks);
 
   /**
    * Notify erasure coder that all the chunks of input blocks are processed so
    * the coder can be able to update internal states, considering next step.
    */
-  public void finish();
+  void finish();
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java → hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java

@@ -18,9 +18,10 @@
 package org.apache.hadoop.io.erasurecode.coder;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.io.erasurecode.ECBlock;
 import org.apache.hadoop.io.erasurecode.ECBlockGroup;
-import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 
 /**
  * An abstract erasure decoder that's to be inherited by new decoders.
@@ -28,14 +29,16 @@ import org.apache.hadoop.io.erasurecode.ECSchema;
  * It implements the {@link ErasureCoder} interface.
  */
 @InterfaceAudience.Private
-public abstract class AbstractErasureDecoder extends AbstractErasureCoder {
-
-  public AbstractErasureDecoder(int numDataUnits, int numParityUnits) {
-    super(numDataUnits, numParityUnits);
-  }
-
-  public AbstractErasureDecoder(ECSchema schema) {
-    super(schema);
+public abstract class ErasureDecoder extends Configured
+    implements ErasureCoder {
+  private final int numDataUnits;
+  private final int numParityUnits;
+  private final ErasureCoderOptions options;
+
+  public ErasureDecoder(ErasureCoderOptions options) {
+    this.options = options;
+    this.numDataUnits = options.getNumDataUnits();
+    this.numParityUnits = options.getNumParityUnits();
   }
 
   @Override
@@ -44,13 +47,20 @@ public abstract class AbstractErasureDecoder extends AbstractErasureCoder {
     return prepareDecodingStep(blockGroup);
   }
 
-  /**
-   * Perform decoding against a block blockGroup.
-   * @param blockGroup
-   * @return decoding step for caller to do the real work
-   */
-  protected abstract ErasureCodingStep prepareDecodingStep(
-      ECBlockGroup blockGroup);
+  @Override
+  public int getNumDataUnits() {
+    return this.numDataUnits;
+  }
+
+  @Override
+  public int getNumParityUnits() {
+    return this.numParityUnits;
+  }
+
+  @Override
+  public ErasureCoderOptions getOptions() {
+    return options;
+  }
 
   /**
    * We have all the data blocks and parity blocks as input blocks for
@@ -96,6 +106,24 @@ public abstract class AbstractErasureDecoder extends AbstractErasureCoder {
     return outputBlocks;
   }
 
+  @Override
+  public boolean preferDirectBuffer() {
+    return false;
+  }
+
+  @Override
+  public void release() {
+    // Nothing to do by default
+  }
+
+  /**
+   * Perform decoding against a block blockGroup.
+   * @param blockGroup
+   * @return decoding step for caller to do the real work
+   */
+  protected abstract ErasureCodingStep prepareDecodingStep(
+      ECBlockGroup blockGroup);
+
   /**
    * Get the number of erased blocks in the block group.
    * @param blockGroup

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java

@@ -27,7 +27,9 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
  * a decoding step involved in the whole process of decoding a block group.
  */
 @InterfaceAudience.Private
-public class ErasureDecodingStep extends AbstractErasureCodingStep {
+public class ErasureDecodingStep implements ErasureCodingStep {
+  private ECBlock[] inputBlocks;
+  private ECBlock[] outputBlocks;
   private int[] erasedIndexes;
   private RawErasureDecoder rawDecoder;
 
@@ -41,7 +43,8 @@ public class ErasureDecodingStep extends AbstractErasureCodingStep {
   public ErasureDecodingStep(ECBlock[] inputBlocks, int[] erasedIndexes,
                              ECBlock[] outputBlocks,
                              RawErasureDecoder rawDecoder) {
-    super(inputBlocks, outputBlocks);
+    this.inputBlocks = inputBlocks;
+    this.outputBlocks = outputBlocks;
     this.erasedIndexes = erasedIndexes;
     this.rawDecoder = rawDecoder;
   }
@@ -51,4 +54,18 @@ public class ErasureDecodingStep extends AbstractErasureCodingStep {
     rawDecoder.decode(inputChunks, erasedIndexes, outputChunks);
   }
 
+  @Override
+  public ECBlock[] getInputBlocks() {
+    return inputBlocks;
+  }
+
+  @Override
+  public ECBlock[] getOutputBlocks() {
+    return outputBlocks;
+  }
+
+  @Override
+  public void finish() {
+    // TODO: Finalize decoder if necessary
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java → hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java

@@ -18,9 +18,10 @@
 package org.apache.hadoop.io.erasurecode.coder;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.io.erasurecode.ECBlock;
 import org.apache.hadoop.io.erasurecode.ECBlockGroup;
-import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 
 /**
  * An abstract erasure encoder that's to be inherited by new encoders.
@@ -28,14 +29,17 @@ import org.apache.hadoop.io.erasurecode.ECSchema;
  * It implements the {@link ErasureCoder} interface.
  */
 @InterfaceAudience.Private
-public abstract class AbstractErasureEncoder extends AbstractErasureCoder {
+public abstract class ErasureEncoder extends Configured
+    implements ErasureCoder {
 
-  public AbstractErasureEncoder(int numDataUnits, int numParityUnits) {
-    super(numDataUnits, numParityUnits);
-  }
+  private final int numDataUnits;
+  private final int numParityUnits;
+  private final ErasureCoderOptions options;
 
-  public AbstractErasureEncoder(ECSchema schema) {
-    super(schema);
+  public ErasureEncoder(ErasureCoderOptions options) {
+    this.options = options;
+    this.numDataUnits = options.getNumDataUnits();
+    this.numParityUnits = options.getNumParityUnits();
   }
 
   @Override
@@ -44,13 +48,20 @@ public abstract class AbstractErasureEncoder extends AbstractErasureCoder {
     return prepareEncodingStep(blockGroup);
   }
 
-  /**
-   * Perform encoding against a block group.
-   * @param blockGroup
-   * @return encoding step for caller to do the real work
-   */
-  protected abstract ErasureCodingStep prepareEncodingStep(
-      ECBlockGroup blockGroup);
+  @Override
+  public int getNumDataUnits() {
+    return numDataUnits;
+  }
+
+  @Override
+  public int getNumParityUnits() {
+    return numParityUnits;
+  }
+
+  @Override
+  public ErasureCoderOptions getOptions() {
+    return options;
+  }
 
   protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) {
     return blockGroup.getDataBlocks();
@@ -59,4 +70,22 @@ public abstract class AbstractErasureEncoder extends AbstractErasureCoder {
   protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) {
     return blockGroup.getParityBlocks();
   }
+
+  @Override
+  public boolean preferDirectBuffer() {
+    return false;
+  }
+
+  @Override
+  public void release() {
+    // Nothing to do by default
+  }
+
+  /**
+   * Perform encoding against a block group.
+   * @param blockGroup
+   * @return encoding step for caller to do the real work
+   */
+  protected abstract ErasureCodingStep prepareEncodingStep(
+      ECBlockGroup blockGroup);
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java

@@ -27,8 +27,9 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
  * an encoding step involved in the whole process of encoding a block group.
  */
 @InterfaceAudience.Private
-public class ErasureEncodingStep extends AbstractErasureCodingStep {
-
+public class ErasureEncodingStep implements ErasureCodingStep {
+  private ECBlock[] inputBlocks;
+  private ECBlock[] outputBlocks;
   private RawErasureEncoder rawEncoder;
 
   /**
@@ -39,7 +40,8 @@ public class ErasureEncodingStep extends AbstractErasureCodingStep {
    */
   public ErasureEncodingStep(ECBlock[] inputBlocks, ECBlock[] outputBlocks,
                              RawErasureEncoder rawEncoder) {
-    super(inputBlocks, outputBlocks);
+    this.inputBlocks = inputBlocks;
+    this.outputBlocks = outputBlocks;
     this.rawEncoder = rawEncoder;
   }
 
@@ -48,4 +50,18 @@ public class ErasureEncodingStep extends AbstractErasureCodingStep {
     rawEncoder.encode(inputChunks, outputChunks);
   }
 
+  @Override
+  public ECBlock[] getInputBlocks() {
+    return inputBlocks;
+  }
+
+  @Override
+  public ECBlock[] getOutputBlocks() {
+    return outputBlocks;
+  }
+
+  @Override
+  public void finish() {
+    rawEncoder.release();
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCodingStep.java → hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHErasureCodingStep.java

@@ -21,28 +21,36 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.erasurecode.ECBlock;
 
 /**
- * Abstract class for common facilities shared by {@link ErasureEncodingStep}
- * and {@link ErasureDecodingStep}.
+ * Abstract class for Hitchhiker common facilities shared by
+ * {@link HHXORErasureEncodingStep}and {@link HHXORErasureDecodingStep}.
  *
- * It implements {@link ErasureEncodingStep}.
+ * It implements {@link ErasureCodingStep}.
  */
 @InterfaceAudience.Private
-public abstract class AbstractErasureCodingStep implements ErasureCodingStep {
+public abstract class HHErasureCodingStep
+        implements ErasureCodingStep {
 
   private ECBlock[] inputBlocks;
   private ECBlock[] outputBlocks;
 
+  private static final int SUB_PACKET_SIZE = 2;
+
   /**
    * Constructor given input blocks and output blocks.
+   *
    * @param inputBlocks
    * @param outputBlocks
    */
-  public AbstractErasureCodingStep(ECBlock[] inputBlocks,
-                                   ECBlock[] outputBlocks) {
+  public HHErasureCodingStep(ECBlock[] inputBlocks,
+                                     ECBlock[] outputBlocks) {
     this.inputBlocks = inputBlocks;
     this.outputBlocks = outputBlocks;
   }
 
+  protected int getSubPacketSize() {
+    return SUB_PACKET_SIZE;
+  }
+
   @Override
   public ECBlock[] getInputBlocks() {
     return inputBlocks;
@@ -55,7 +63,6 @@ public abstract class AbstractErasureCodingStep implements ErasureCodingStep {
 
   @Override
   public void finish() {
-    // NOOP by default
+    // TODO: Finalize encoder/decoder if necessary
   }
-
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java

@@ -21,7 +21,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.erasurecode.CodecUtil;
 import org.apache.hadoop.io.erasurecode.ECBlock;
 import org.apache.hadoop.io.erasurecode.ECBlockGroup;
-import org.apache.hadoop.io.erasurecode.ECSchema;
 import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
@@ -39,16 +38,12 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
  * This is Hitchhiker-XOR erasure decoder that decodes a block group.
  */
 @InterfaceAudience.Private
-public class HHXORErasureDecoder extends AbstractErasureDecoder {
+public class HHXORErasureDecoder extends ErasureDecoder {
   private RawErasureDecoder rsRawDecoder;
   private RawErasureEncoder xorRawEncoder;
 
-  public HHXORErasureDecoder(int numDataUnits, int numParityUnits) {
-    super(numDataUnits, numParityUnits);
-  }
-
-  public HHXORErasureDecoder(ECSchema schema) {
-    super(schema);
+  public HHXORErasureDecoder(ErasureCoderOptions options) {
+    super(options);
   }
 
   @Override
@@ -71,24 +66,25 @@ public class HHXORErasureDecoder extends AbstractErasureDecoder {
 
   private RawErasureDecoder checkCreateRSRawDecoder() {
     if (rsRawDecoder == null) {
-      ErasureCoderOptions coderOptions = new ErasureCoderOptions(
-          getNumDataUnits(), getNumParityUnits());
       rsRawDecoder = CodecUtil.createRawDecoder(getConf(),
-              ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
+              ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions());
     }
     return rsRawDecoder;
   }
 
   private RawErasureEncoder checkCreateXorRawEncoder() {
     if (xorRawEncoder == null) {
-      ErasureCoderOptions coderOptions = new ErasureCoderOptions(
-          getNumDataUnits(), getNumParityUnits());
       xorRawEncoder = CodecUtil.createRawEncoder(getConf(),
-          ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
+          ErasureCodeConstants.XOR_CODEC_NAME, getOptions());
     }
     return xorRawEncoder;
   }
 
+  @Override
+  public boolean preferDirectBuffer() {
+    return false;
+  }
+
   @Override
   public void release() {
     if (rsRawDecoder != null) {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java

@@ -32,7 +32,7 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
  * decoding a block group.
  */
 @InterfaceAudience.Private
-public class HHXORErasureDecodingStep extends AbstractHHErasureCodingStep {
+public class HHXORErasureDecodingStep extends HHErasureCodingStep {
   private int pbIndex;
   private int[] piggyBackIndex;
   private int[] piggyBackFullIndex;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java

@@ -21,7 +21,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.erasurecode.CodecUtil;
 import org.apache.hadoop.io.erasurecode.ECBlock;
 import org.apache.hadoop.io.erasurecode.ECBlockGroup;
-import org.apache.hadoop.io.erasurecode.ECSchema;
 import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
@@ -38,16 +37,12 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
  * This is Hitchhiker-XOR erasure encoder that encodes a block group.
  */
 @InterfaceAudience.Private
-public class HHXORErasureEncoder extends AbstractErasureEncoder {
+public class HHXORErasureEncoder extends ErasureEncoder {
   private RawErasureEncoder rsRawEncoder;
   private RawErasureEncoder xorRawEncoder;
 
-  public HHXORErasureEncoder(int numDataUnits, int numParityUnits) {
-    super(numDataUnits, numParityUnits);
-  }
-
-  public HHXORErasureEncoder(ECSchema schema) {
-    super(schema);
+  public HHXORErasureEncoder(ErasureCoderOptions options) {
+    super(options);
   }
 
   @Override
@@ -65,21 +60,17 @@ public class HHXORErasureEncoder extends AbstractErasureEncoder {
 
   private RawErasureEncoder checkCreateRSRawEncoder() {
     if (rsRawEncoder == null) {
-      ErasureCoderOptions coderOptions = new ErasureCoderOptions(
-          getNumDataUnits(), getNumParityUnits());
       rsRawEncoder = CodecUtil.createRawEncoder(getConf(),
-          ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
+          ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions());
     }
     return rsRawEncoder;
   }
 
   private RawErasureEncoder checkCreateXorRawEncoder() {
     if (xorRawEncoder == null) {
-      ErasureCoderOptions erasureCoderOptions = new ErasureCoderOptions(
-          getNumDataUnits(), getNumParityUnits());
       xorRawEncoder = CodecUtil.createRawEncoder(getConf(),
           ErasureCodeConstants.XOR_CODEC_NAME,
-          erasureCoderOptions);
+          getOptions());
     }
     return xorRawEncoder;
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java

@@ -31,7 +31,7 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
  * encoding a block group.
  */
 @InterfaceAudience.Private
-public class HHXORErasureEncodingStep extends AbstractHHErasureCodingStep {
+public class HHXORErasureEncodingStep extends HHErasureCodingStep {
   private int[] piggyBackIndex;
   private RawErasureEncoder rsRawEncoder;
   private RawErasureEncoder xorRawEncoder;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java

@@ -21,7 +21,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.erasurecode.CodecUtil;
 import org.apache.hadoop.io.erasurecode.ECBlock;
 import org.apache.hadoop.io.erasurecode.ECBlockGroup;
-import org.apache.hadoop.io.erasurecode.ECSchema;
 import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
@@ -32,15 +31,11 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
  * It implements {@link ErasureCoder}.
  */
 @InterfaceAudience.Private
-public class RSErasureDecoder extends AbstractErasureDecoder {
+public class RSErasureDecoder extends ErasureDecoder {
   private RawErasureDecoder rsRawDecoder;
 
-  public RSErasureDecoder(int numDataUnits, int numParityUnits) {
-    super(numDataUnits, numParityUnits);
-  }
-
-  public RSErasureDecoder(ECSchema schema) {
-    super(schema);
+  public RSErasureDecoder(ErasureCoderOptions options) {
+    super(options);
   }
 
   @Override
@@ -56,11 +51,8 @@ public class RSErasureDecoder extends AbstractErasureDecoder {
 
   private RawErasureDecoder checkCreateRSRawDecoder() {
     if (rsRawDecoder == null) {
-      // TODO: we should create the raw coder according to codec.
-      ErasureCoderOptions coderOptions = new ErasureCoderOptions(
-          getNumDataUnits(), getNumParityUnits());
       rsRawDecoder = CodecUtil.createRawDecoder(getConf(),
-          ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
+          ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions());
     }
     return rsRawDecoder;
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java

@@ -21,7 +21,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.erasurecode.CodecUtil;
 import org.apache.hadoop.io.erasurecode.ECBlock;
 import org.apache.hadoop.io.erasurecode.ECBlockGroup;
-import org.apache.hadoop.io.erasurecode.ECSchema;
 import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
@@ -32,15 +31,11 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
  * It implements {@link ErasureCoder}.
  */
 @InterfaceAudience.Private
-public class RSErasureEncoder extends AbstractErasureEncoder {
+public class RSErasureEncoder extends ErasureEncoder {
   private RawErasureEncoder rawEncoder;
 
-  public RSErasureEncoder(int numDataUnits, int numParityUnits) {
-    super(numDataUnits, numParityUnits);
-  }
-
-  public RSErasureEncoder(ECSchema schema) {
-    super(schema);
+  public RSErasureEncoder(ErasureCoderOptions options) {
+    super(options);
   }
 
   @Override
@@ -57,10 +52,8 @@ public class RSErasureEncoder extends AbstractErasureEncoder {
   private RawErasureEncoder checkCreateRSRawEncoder() {
     if (rawEncoder == null) {
       // TODO: we should create the raw coder according to codec.
-      ErasureCoderOptions coderOptions = new ErasureCoderOptions(
-          getNumDataUnits(), getNumParityUnits());
       rawEncoder = CodecUtil.createRawEncoder(getConf(),
-          ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
+          ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions());
     }
     return rawEncoder;
   }
@@ -71,4 +64,9 @@ public class RSErasureEncoder extends AbstractErasureEncoder {
       rawEncoder.release();
     }
   }
+
+  @Override
+  public boolean preferDirectBuffer() {
+    return false;
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java

@@ -21,7 +21,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.erasurecode.CodecUtil;
 import org.apache.hadoop.io.erasurecode.ECBlock;
 import org.apache.hadoop.io.erasurecode.ECBlockGroup;
-import org.apache.hadoop.io.erasurecode.ECSchema;
 import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
@@ -32,23 +31,17 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
  * It implements {@link ErasureCoder}.
  */
 @InterfaceAudience.Private
-public class XORErasureDecoder extends AbstractErasureDecoder {
+public class XORErasureDecoder extends ErasureDecoder {
 
-  public XORErasureDecoder(int numDataUnits, int numParityUnits) {
-    super(numDataUnits, numParityUnits);
-  }
-
-  public XORErasureDecoder(ECSchema schema) {
-    super(schema);
+  public XORErasureDecoder(ErasureCoderOptions options) {
+    super(options);
   }
 
   @Override
   protected ErasureCodingStep prepareDecodingStep(
       final ECBlockGroup blockGroup) {
-    ErasureCoderOptions coderOptions = new ErasureCoderOptions(
-        getNumDataUnits(), getNumParityUnits());
     RawErasureDecoder rawDecoder = CodecUtil.createRawDecoder(getConf(),
-        ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
+        ErasureCodeConstants.XOR_CODEC_NAME, getOptions());
 
     ECBlock[] inputBlocks = getInputBlocks(blockGroup);
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java

@@ -21,7 +21,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.erasurecode.CodecUtil;
 import org.apache.hadoop.io.erasurecode.ECBlock;
 import org.apache.hadoop.io.erasurecode.ECBlockGroup;
-import org.apache.hadoop.io.erasurecode.ECSchema;
 import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
@@ -32,28 +31,21 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
  * It implements {@link ErasureCoder}.
  */
 @InterfaceAudience.Private
-public class XORErasureEncoder extends AbstractErasureEncoder {
+public class XORErasureEncoder extends ErasureEncoder {
 
-  public XORErasureEncoder(int numDataUnits, int numParityUnits) {
-    super(numDataUnits, numParityUnits);
-  }
-
-  public XORErasureEncoder(ECSchema schema) {
-    super(schema);
+  public XORErasureEncoder(ErasureCoderOptions options) {
+    super(options);
   }
 
   @Override
   protected ErasureCodingStep prepareEncodingStep(
       final ECBlockGroup blockGroup) {
-    ErasureCoderOptions coderOptions = new ErasureCoderOptions(
-        getNumDataUnits(), getNumParityUnits());
     RawErasureEncoder rawEncoder = CodecUtil.createRawEncoder(getConf(),
-        ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
+        ErasureCodeConstants.XOR_CODEC_NAME, getOptions());
 
     ECBlock[] inputBlocks = getInputBlocks(blockGroup);
 
     return new ErasureEncodingStep(inputBlocks,
         getOutputBlocks(blockGroup), rawEncoder);
   }
-
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractHHErasureCodingStep.java → hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/package-info.java

@@ -15,35 +15,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.io.erasurecode.coder;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.erasurecode.ECBlock;
 
 /**
- * Abstract class for Hitchhiker common facilities shared by
- * {@link HHXORErasureEncodingStep}and {@link HHXORErasureDecodingStep}.
- *
- * It implements {@link AbstractErasureCodingStep}.
+ * Erasure coders framework.
  */
 @InterfaceAudience.Private
-public abstract class AbstractHHErasureCodingStep
-        extends AbstractErasureCodingStep {
-
-  private static final int SUB_PACKET_SIZE = 2;
+@InterfaceStability.Unstable
+package org.apache.hadoop.io.erasurecode.coder;
 
-  /**
-   * Constructor given input blocks and output blocks.
-   *
-   * @param inputBlocks
-   * @param outputBlocks
-   */
-  public AbstractHHErasureCodingStep(ECBlock[] inputBlocks,
-                                     ECBlock[] outputBlocks) {
-    super(inputBlocks, outputBlocks);
-  }
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
 
-  protected int getSubPacketSize() {
-    return SUB_PACKET_SIZE;
-  }
-}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderUtil.java

@@ -28,7 +28,7 @@ import java.util.Arrays;
  * Helpful utilities for implementing some raw erasure coders.
  */
 @InterfaceAudience.Private
-final class CoderUtil {
+public final class CoderUtil {
 
   private CoderUtil() {
     // No called

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java

@@ -29,6 +29,7 @@ import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.hadoop.security.authentication.server.AuthenticationHandler;
 import org.apache.hadoop.security.authentication.server.AuthenticationToken;
 import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
+import org.apache.hadoop.security.authentication.server.MultiSchemeAuthenticationHandler;
 import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
 import org.apache.hadoop.security.authentication.util.ZKSignerSecretProvider;
 import org.apache.hadoop.security.authorize.AuthorizationException;
@@ -38,7 +39,6 @@ import org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManage
 import org.apache.hadoop.util.HttpExceptionUtils;
 import org.apache.http.NameValuePair;
 import org.apache.http.client.utils.URLEncodedUtils;
-import org.codehaus.jackson.map.ObjectMapper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -51,13 +51,10 @@ import javax.servlet.http.HttpServletRequestWrapper;
 import javax.servlet.http.HttpServletResponse;
 
 import java.io.IOException;
-import java.io.Writer;
 import java.nio.charset.Charset;
 import java.security.Principal;
 import java.util.Enumeration;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Properties;
 
 /**
@@ -142,6 +139,9 @@ public class DelegationTokenAuthenticationFilter
     } else if (authType.equals(KerberosAuthenticationHandler.TYPE)) {
       props.setProperty(AUTH_TYPE,
           KerberosDelegationTokenAuthenticationHandler.class.getName());
+    } else if (authType.equals(MultiSchemeAuthenticationHandler.TYPE)) {
+      props.setProperty(AUTH_TYPE,
+          MultiSchemeDelegationTokenAuthenticationHandler.class.getName());
     }
   }
 
@@ -307,5 +307,4 @@ public class DelegationTokenAuthenticationFilter
       }
     }
   }
-
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java

@@ -112,6 +112,10 @@ public abstract class DelegationTokenAuthenticationHandler
     return tokenManager;
   }
 
+  AuthenticationHandler getAuthHandler() {
+    return authHandler;
+  }
+
   @Override
   public void init(Properties config) throws ServletException {
     authHandler.init(config);
@@ -162,6 +166,24 @@ public abstract class DelegationTokenAuthenticationHandler
 
   private static final String ENTER = System.getProperty("line.separator");
 
+  /**
+   * This method checks if the given HTTP request corresponds to a management
+   * operation.
+   *
+   * @param request The HTTP request
+   * @return true if the given HTTP request corresponds to a management
+   *         operation false otherwise
+   * @throws IOException In case of I/O error.
+   */
+  protected final boolean isManagementOperation(HttpServletRequest request)
+      throws IOException {
+    String op = ServletUtils.getParameter(request,
+        KerberosDelegationTokenAuthenticator.OP_PARAM);
+    op = (op != null) ? StringUtils.toUpperCase(op) : null;
+    return DELEGATION_TOKEN_OPS.contains(op) &&
+        !request.getMethod().equals("OPTIONS");
+  }
+
   @Override
   @SuppressWarnings("unchecked")
   public boolean managementOperation(AuthenticationToken token,
@@ -171,8 +193,7 @@ public abstract class DelegationTokenAuthenticationHandler
     String op = ServletUtils.getParameter(request,
         KerberosDelegationTokenAuthenticator.OP_PARAM);
     op = (op != null) ? StringUtils.toUpperCase(op) : null;
-    if (DELEGATION_TOKEN_OPS.contains(op) &&
-        !request.getMethod().equals("OPTIONS")) {
+    if (isManagementOperation(request)) {
       KerberosDelegationTokenAuthenticator.DelegationTokenOperation dtOp =
           KerberosDelegationTokenAuthenticator.
               DelegationTokenOperation.valueOf(op);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/MultiSchemeDelegationTokenAuthenticationHandler.java

@@ -0,0 +1,182 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.security.token.delegation.web;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Set;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.server.AuthenticationHandlerUtil;
+import org.apache.hadoop.security.authentication.server.AuthenticationToken;
+import org.apache.hadoop.security.authentication.server.CompositeAuthenticationHandler;
+import org.apache.hadoop.security.authentication.server.HttpConstants;
+import org.apache.hadoop.security.authentication.server.MultiSchemeAuthenticationHandler;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+
+/**
+ * A {@link CompositeAuthenticationHandler} that supports multiple HTTP
+ * authentication schemes along with Delegation Token functionality. e.g.
+ * server can support multiple authentication mechanisms such as Kerberos
+ * (SPENGO) and LDAP. During the authentication phase, server will specify
+ * all possible authentication schemes and let client choose the appropriate
+ * scheme. Please refer to RFC-2616 and HADOOP-12082 for more details.
+ *
+ * Internally it uses {@link MultiSchemeAuthenticationHandler} implementation.
+ * This handler also provides an option to enable delegation token management
+ * functionality for only a specified subset of authentication schemes. This is
+ * required to ensure that only schemes with strongest level of security should
+ * be used for delegation token management.
+ *
+ * <p/>
+ * In addition to the wrapped {@link AuthenticationHandler} configuration
+ * properties, this handler supports the following properties prefixed with the
+ * type of the wrapped <code>AuthenticationHandler</code>:
+ * <ul>
+ * <li>delegation-token.token-kind: the token kind for generated tokens (no
+ * default, required property).</li>
+ * <li>delegation-token.update-interval.sec: secret manager master key update
+ * interval in seconds (default 1 day).</li>
+ * <li>delegation-token.max-lifetime.sec: maximum life of a delegation token in
+ * seconds (default 7 days).</li>
+ * <li>delegation-token.renewal-interval.sec: renewal interval for delegation
+ * tokens in seconds (default 1 day).</li>
+ * <li>delegation-token.removal-scan-interval.sec: delegation tokens removal
+ * scan interval in seconds (default 1 hour).</li>
+ * <li>delegation.http.schemes: A comma separated list of HTTP authentication
+ * mechanisms (e.g. Negotiate, Basic) etc. to be allowed for delegation token
+ * management operations.</li>
+ * </ul>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class MultiSchemeDelegationTokenAuthenticationHandler extends
+    DelegationTokenAuthenticationHandler implements
+    CompositeAuthenticationHandler {
+
+  public static final String DELEGATION_TOKEN_SCHEMES_PROPERTY =
+      "multi-scheme-auth-handler.delegation.schemes";
+  private static final Splitter STR_SPLITTER = Splitter.on(',').trimResults()
+      .omitEmptyStrings();
+
+  private Set<String> delegationAuthSchemes = null;
+
+  public MultiSchemeDelegationTokenAuthenticationHandler() {
+    super(new MultiSchemeAuthenticationHandler(
+        MultiSchemeAuthenticationHandler.TYPE + TYPE_POSTFIX));
+  }
+
+  @Override
+  public Collection<String> getTokenTypes() {
+    return ((CompositeAuthenticationHandler) getAuthHandler()).getTokenTypes();
+  }
+
+  @Override
+  public void init(Properties config) throws ServletException {
+    super.init(config);
+
+    // Figure out the HTTP authentication schemes configured.
+    String schemesProperty =
+        Preconditions.checkNotNull(config
+            .getProperty(MultiSchemeAuthenticationHandler.SCHEMES_PROPERTY));
+
+    // Figure out the HTTP authentication schemes configured for delegation
+    // tokens.
+    String delegationAuthSchemesProp =
+        Preconditions.checkNotNull(config
+            .getProperty(DELEGATION_TOKEN_SCHEMES_PROPERTY));
+
+    Set<String> authSchemes = new HashSet<>();
+    for (String scheme : STR_SPLITTER.split(schemesProperty)) {
+      authSchemes.add(AuthenticationHandlerUtil.checkAuthScheme(scheme));
+    }
+
+    delegationAuthSchemes = new HashSet<>();
+    for (String scheme : STR_SPLITTER.split(delegationAuthSchemesProp)) {
+      delegationAuthSchemes.add(AuthenticationHandlerUtil
+          .checkAuthScheme(scheme));
+    }
+
+    Preconditions.checkArgument(authSchemes.containsAll(delegationAuthSchemes));
+  }
+
+  /**
+   * This method is overridden to restrict HTTP authentication schemes
+   * available for delegation token management functionality. The
+   * authentication schemes to be used for delegation token management are
+   * configured using {@link DELEGATION_TOKEN_SCHEMES_PROPERTY}
+   *
+   * The basic logic here is to check if the current request is for delegation
+   * token management. If yes then check if the request contains an
+   * "Authorization" header. If it is missing, then return the HTTP 401
+   * response with WWW-Authenticate header for each scheme configured for
+   * delegation token management.
+   *
+   * It is also possible for a client to preemptively send Authorization header
+   * for a scheme not configured for delegation token management. We detect
+   * this case and return the HTTP 401 response with WWW-Authenticate header
+   * for each scheme configured for delegation token management.
+   *
+   * If a client has sent a request with "Authorization" header for a scheme
+   * configured for delegation token management, then it is forwarded to
+   * underlying {@link MultiSchemeAuthenticationHandler} for actual
+   * authentication.
+   *
+   * Finally all other requests (excluding delegation token management) are
+   * forwarded to underlying {@link MultiSchemeAuthenticationHandler} for
+   * actual authentication.
+   */
+  @Override
+  public AuthenticationToken authenticate(HttpServletRequest request,
+      HttpServletResponse response)
+          throws IOException, AuthenticationException {
+    String authorization =
+        request.getHeader(HttpConstants.AUTHORIZATION_HEADER);
+
+    if (isManagementOperation(request)) {
+      boolean schemeConfigured = false;
+      if (authorization != null) {
+        for (String scheme : delegationAuthSchemes) {
+          if (AuthenticationHandlerUtil.
+              matchAuthScheme(scheme, authorization)) {
+            schemeConfigured = true;
+            break;
+          }
+        }
+      }
+      if (!schemeConfigured) {
+        response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+        for (String scheme : delegationAuthSchemes) {
+          response.addHeader(WWW_AUTHENTICATE, scheme);
+        }
+        return null;
+      }
+    }
+
+    return super.authenticate(request, response);
+  }
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java

@@ -20,9 +20,6 @@ package org.apache.hadoop.util;
 
 import java.io.File;
 import java.io.IOException;
-import java.nio.file.DirectoryStream;
-import java.nio.file.DirectoryIteratorException;
-import java.nio.file.Files;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -53,62 +50,6 @@ public class DiskChecker {
     }
   }
       
-  /** 
-   * The semantics of mkdirsWithExistsCheck method is different from the mkdirs
-   * method provided in the Sun's java.io.File class in the following way:
-   * While creating the non-existent parent directories, this method checks for
-   * the existence of those directories if the mkdir fails at any point (since
-   * that directory might have just been created by some other process).
-   * If both mkdir() and the exists() check fails for any seemingly 
-   * non-existent directory, then we signal an error; Sun's mkdir would signal
-   * an error (return false) if a directory it is attempting to create already
-   * exists or the mkdir fails.
-   * @param dir
-   * @return true on success, false on failure
-   */
-  public static boolean mkdirsWithExistsCheck(File dir) {
-    if (dir.mkdir() || dir.exists()) {
-      return true;
-    }
-    File canonDir = null;
-    try {
-      canonDir = dir.getCanonicalFile();
-    } catch (IOException e) {
-      return false;
-    }
-    String parent = canonDir.getParent();
-    return (parent != null) && 
-           (mkdirsWithExistsCheck(new File(parent)) &&
-                                      (canonDir.mkdir() || canonDir.exists()));
-  }
-
-  /**
-   * Recurse down a directory tree, checking all child directories.
-   * @param dir
-   * @throws DiskErrorException
-   */
-  public static void checkDirs(File dir) throws DiskErrorException {
-    checkDir(dir);
-    IOException ex = null;
-    try (DirectoryStream<java.nio.file.Path> stream =
-        Files.newDirectoryStream(dir.toPath())) {
-      for (java.nio.file.Path entry: stream) {
-        File child = entry.toFile();
-        if (child.isDirectory()) {
-          checkDirs(child);
-        }
-      }
-    } catch (DirectoryIteratorException de) {
-      ex = de.getCause();
-    } catch (IOException ie) {
-      ex = ie;
-    }
-    if (ex != null) {
-      throw new DiskErrorException("I/O error when open a directory: "
-          + dir.toString(), ex);
-    }
-  }
-
   /**
    * Create the directory if it doesn't exist and check that dir is readable,
    * writable and executable
@@ -121,39 +62,7 @@ public class DiskChecker {
       throw new DiskErrorException("Cannot create directory: "
                                    + dir.toString());
     }
-    checkDirAccess(dir);
-  }
-
-  /**
-   * Create the directory or check permissions if it already exists.
-   *
-   * The semantics of mkdirsWithExistsAndPermissionCheck method is different
-   * from the mkdirs method provided in the Sun's java.io.File class in the
-   * following way:
-   * While creating the non-existent parent directories, this method checks for
-   * the existence of those directories if the mkdir fails at any point (since
-   * that directory might have just been created by some other process).
-   * If both mkdir() and the exists() check fails for any seemingly
-   * non-existent directory, then we signal an error; Sun's mkdir would signal
-   * an error (return false) if a directory it is attempting to create already
-   * exists or the mkdir fails.
-   *
-   * @param localFS local filesystem
-   * @param dir directory to be created or checked
-   * @param expected expected permission
-   * @throws IOException
-   */
-  public static void mkdirsWithExistsAndPermissionCheck(
-      LocalFileSystem localFS, Path dir, FsPermission expected)
-      throws IOException {
-    File directory = localFS.pathToFile(dir);
-    boolean created = false;
-
-    if (!directory.exists())
-      created = mkdirsWithExistsCheck(directory);
-
-    if (created || !localFS.getFileStatus(dir).getPermission().equals(expected))
-        localFS.setPermission(dir, expected);
+    checkAccessByFileMethods(dir);
   }
 
   /**
@@ -170,24 +79,7 @@ public class DiskChecker {
                               FsPermission expected)
   throws DiskErrorException, IOException {
     mkdirsWithExistsAndPermissionCheck(localFS, dir, expected);
-    checkDirAccess(localFS.pathToFile(dir));
-  }
-
-  /**
-   * Checks that the given file is a directory and that the current running
-   * process can read, write, and execute it.
-   * 
-   * @param dir File to check
-   * @throws DiskErrorException if dir is not a directory, not readable, not
-   *   writable, or not executable
-   */
-  private static void checkDirAccess(File dir) throws DiskErrorException {
-    if (!dir.isDirectory()) {
-      throw new DiskErrorException("Not a directory: "
-                                   + dir.toString());
-    }
-
-    checkAccessByFileMethods(dir);
+    checkAccessByFileMethods(localFS.pathToFile(dir));
   }
 
   /**
@@ -200,6 +92,11 @@ public class DiskChecker {
    */
   private static void checkAccessByFileMethods(File dir)
       throws DiskErrorException {
+    if (!dir.isDirectory()) {
+      throw new DiskErrorException("Not a directory: "
+          + dir.toString());
+    }
+
     if (!FileUtil.canRead(dir)) {
       throw new DiskErrorException("Directory is not readable: "
                                    + dir.toString());
@@ -215,4 +112,65 @@ public class DiskChecker {
                                    + dir.toString());
     }
   }
+
+  /**
+   * The semantics of mkdirsWithExistsCheck method is different from the mkdirs
+   * method provided in the Sun's java.io.File class in the following way:
+   * While creating the non-existent parent directories, this method checks for
+   * the existence of those directories if the mkdir fails at any point (since
+   * that directory might have just been created by some other process).
+   * If both mkdir() and the exists() check fails for any seemingly
+   * non-existent directory, then we signal an error; Sun's mkdir would signal
+   * an error (return false) if a directory it is attempting to create already
+   * exists or the mkdir fails.
+   * @param dir
+   * @return true on success, false on failure
+   */
+  private static boolean mkdirsWithExistsCheck(File dir) {
+    if (dir.mkdir() || dir.exists()) {
+      return true;
+    }
+    File canonDir;
+    try {
+      canonDir = dir.getCanonicalFile();
+    } catch (IOException e) {
+      return false;
+    }
+    String parent = canonDir.getParent();
+    return (parent != null) &&
+        (mkdirsWithExistsCheck(new File(parent)) &&
+            (canonDir.mkdir() || canonDir.exists()));
+  }
+
+  /**
+   * Create the directory or check permissions if it already exists.
+   *
+   * The semantics of mkdirsWithExistsAndPermissionCheck method is different
+   * from the mkdirs method provided in the Sun's java.io.File class in the
+   * following way:
+   * While creating the non-existent parent directories, this method checks for
+   * the existence of those directories if the mkdir fails at any point (since
+   * that directory might have just been created by some other process).
+   * If both mkdir() and the exists() check fails for any seemingly
+   * non-existent directory, then we signal an error; Sun's mkdir would signal
+   * an error (return false) if a directory it is attempting to create already
+   * exists or the mkdir fails.
+   *
+   * @param localFS local filesystem
+   * @param dir directory to be created or checked
+   * @param expected expected permission
+   * @throws IOException
+   */
+  static void mkdirsWithExistsAndPermissionCheck(
+      LocalFileSystem localFS, Path dir, FsPermission expected)
+      throws IOException {
+    File directory = localFS.pathToFile(dir);
+    boolean created = false;
+
+    if (!directory.exists())
+      created = mkdirsWithExistsCheck(directory);
+
+    if (created || !localFS.getFileStatus(dir).getPermission().equals(expected))
+      localFS.setPermission(dir, expected);
+  }
 }

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/dump.c

@@ -57,11 +57,11 @@ void dumpCodingMatrix(unsigned char* buf, int n1, int n2) {
 
 void dumpEncoder(IsalEncoder* pCoder) {
   int numDataUnits = pCoder->coder.numDataUnits;
-  int numParityUnits = pCoder->coder.numDataUnits;
+  int numParityUnits = pCoder->coder.numParityUnits;
   int numAllUnits = pCoder->coder.numAllUnits;
 
-  printf("Encoding (numAlnumParityUnitslUnits = %d, numDataUnits = %d)\n",
-                                    numParityUnits, numDataUnits);
+  printf("Encoding (numAllUnits = %d, numParityUnits = %d, numDataUnits = %d)\n",
+                                    numAllUnits, numParityUnits, numDataUnits);
 
   printf("\n\nEncodeMatrix:\n");
   dumpCodingMatrix((unsigned char*) pCoder->encodeMatrix,
@@ -91,7 +91,7 @@ void dumpDecoder(IsalDecoder* pCoder) {
 
   printf("InvertMatrix:\n");
   dumpCodingMatrix((unsigned char*) pCoder->invertMatrix,
-                                   numDataUnits, numDataUnits);
+                                   numDataUnits, numAllUnits);
 
   printf("DecodeMatrix:\n");
   dumpCodingMatrix((unsigned char*) pCoder->decodeMatrix,

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.h

@@ -57,7 +57,7 @@ typedef void (*__d_ec_encode_data_update)(int, int, int, int, unsigned char*,
 #endif
 
 #ifdef WINDOWS
-// For erasure_code.h
+// For gf_util.h
 typedef unsigned char (__cdecl *__d_gf_mul)(unsigned char, unsigned char);
 typedef unsigned char (__cdecl *__d_gf_inv)(unsigned char);
 typedef void (__cdecl *__d_gf_gen_rs_matrix)(unsigned char *, int, int);

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -994,8 +994,8 @@
 <property>
   <name>fs.s3a.threads.max</name>
   <value>10</value>
-  <description> Maximum number of concurrent active (part)uploads,
-    which each use a thread from the threadpool.</description>
+  <description>The total number of threads available in the filesystem for data
+    uploads *or any other queued filesystem operation*.</description>
 </property>
 
 <property>
@@ -1008,8 +1008,7 @@
 <property>
   <name>fs.s3a.max.total.tasks</name>
   <value>5</value>
-  <description>Number of (part)uploads allowed to the queue before
-    blocking additional uploads.</description>
+  <description>The number of operations which can be queued for execution</description>
 </property>
 
 <property>
@@ -1047,13 +1046,21 @@
   <name>fs.s3a.multipart.purge</name>
   <value>false</value>
   <description>True if you want to purge existing multipart uploads that may not have been
-     completed/aborted correctly</description>
+    completed/aborted correctly. The corresponding purge age is defined in
+    fs.s3a.multipart.purge.age.
+    If set, when the filesystem is instantiated then all outstanding uploads
+    older than the purge age will be terminated -across the entire bucket.
+    This will impact multipart uploads by other applications and users. so should
+    be used sparingly, with an age value chosen to stop failed uploads, without
+    breaking ongoing operations.
+  </description>
 </property>
 
 <property>
   <name>fs.s3a.multipart.purge.age</name>
   <value>86400</value>
-  <description>Minimum age in seconds of multipart uploads to purge</description>
+  <description>Minimum age in seconds of multipart uploads to purge.
+  </description>
 </property>
 
 <property>
@@ -1086,10 +1093,50 @@
 <property>
   <name>fs.s3a.fast.upload</name>
   <value>false</value>
-  <description>Upload directly from memory instead of buffering to
-    disk first. Memory usage and parallelism can be controlled as up to
-    fs.s3a.multipart.size memory is consumed for each (part)upload actively
-    uploading (fs.s3a.threads.max) or queueing (fs.s3a.max.total.tasks)</description>
+  <description>
+    Use the incremental block-based fast upload mechanism with
+    the buffering mechanism set in fs.s3a.fast.upload.buffer.
+  </description>
+</property>
+
+<property>
+  <name>fs.s3a.fast.upload.buffer</name>
+  <value>disk</value>
+  <description>
+    The buffering mechanism to use when using S3A fast upload
+    (fs.s3a.fast.upload=true). Values: disk, array, bytebuffer.
+    This configuration option has no effect if fs.s3a.fast.upload is false.
+
+    "disk" will use the directories listed in fs.s3a.buffer.dir as
+    the location(s) to save data prior to being uploaded.
+
+    "array" uses arrays in the JVM heap
+
+    "bytebuffer" uses off-heap memory within the JVM.
+
+    Both "array" and "bytebuffer" will consume memory in a single stream up to the number
+    of blocks set by:
+
+        fs.s3a.multipart.size * fs.s3a.fast.upload.active.blocks.
+
+    If using either of these mechanisms, keep this value low
+
+    The total number of threads performing work across all threads is set by
+    fs.s3a.threads.max, with fs.s3a.max.total.tasks values setting the number of queued
+    work items.
+  </description>
+</property>
+
+<property>
+  <name>fs.s3a.fast.upload.active.blocks</name>
+  <value>4</value>
+  <description>
+    Maximum Number of blocks a single output stream can have
+    active (uploading, or queued to the central FileSystem
+    instance's pool of queued operations.
+
+    This stops a single stream overloading the shared thread pool.
+  </description>
 </property>
 
 <property>
@@ -1100,13 +1147,6 @@
   any call to setReadahead() is made to an open stream.</description>
 </property>
 
-<property>
-  <name>fs.s3a.fast.buffer.size</name>
-  <value>1048576</value>
-  <description>Size of initial memory buffer in bytes allocated for an
-    upload. No effect if fs.s3a.fast.upload is false.</description>
-</property>
-
 <property>
   <name>fs.s3a.user.agent.prefix</name>
   <value></value>

hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md

@@ -35,7 +35,7 @@ Installation
 
 Installing a Hadoop cluster typically involves unpacking the software on all the machines in the cluster or installing it via a packaging system as appropriate for your operating system. It is important to divide up the hardware into functions.
 
-Typically one machine in the cluster is designated as the NameNode and another machine the as ResourceManager, exclusively. These are the masters. Other services (such as Web App Proxy Server and MapReduce Job History server) are usually run either on dedicated hardware or on shared infrastrucutre, depending upon the load.
+Typically one machine in the cluster is designated as the NameNode and another machine as the ResourceManager, exclusively. These are the masters. Other services (such as Web App Proxy Server and MapReduce Job History server) are usually run either on dedicated hardware or on shared infrastructure, depending upon the load.
 
 The rest of the machines in the cluster act as both DataNode and NodeManager. These are the workers.
 

hadoop-common-project/hadoop-common/src/site/markdown/Compatibility.md

@@ -68,7 +68,7 @@ Wire compatibility concerns data being transmitted over the wire between Hadoop
 #### Use Cases
 
 * Client-Server compatibility is required to allow users to continue using the old clients even after upgrading the server (cluster) to a later version (or vice versa). For example, a Hadoop 2.1.0 client talking to a Hadoop 2.3.0 cluster.
-* Client-Server compatibility is also required to allow users to upgrade the client before upgrading the server (cluster). For example, a Hadoop 2.4.0 client talking to a Hadoop 2.3.0 cluster. This allows deployment of client-side bug fixes ahead of full cluster upgrades. Note that new cluster features invoked by new client APIs or shell commands will not be usable. YARN applications that attempt to use new APIs (including new fields in data structures) that have not yet deployed to the cluster can expect link exceptions.
+* Client-Server compatibility is also required to allow users to upgrade the client before upgrading the server (cluster). For example, a Hadoop 2.4.0 client talking to a Hadoop 2.3.0 cluster. This allows deployment of client-side bug fixes ahead of full cluster upgrades. Note that new cluster features invoked by new client APIs or shell commands will not be usable. YARN applications that attempt to use new APIs (including new fields in data structures) that have not yet been deployed to the cluster can expect link exceptions.
 * Client-Server compatibility is also required to allow upgrading individual components without upgrading others. For example, upgrade HDFS from version 2.1.0 to 2.2.0 without upgrading MapReduce.
 * Server-Server compatibility is required to allow mixed versions within an active cluster so the cluster may be upgraded without downtime in a rolling fashion.
 
@@ -76,7 +76,7 @@ Wire compatibility concerns data being transmitted over the wire between Hadoop
 
 * Both Client-Server and Server-Server compatibility is preserved within a major release. (Different policies for different categories are yet to be considered.)
 * Compatibility can be broken only at a major release, though breaking compatibility even at major releases has grave consequences and should be discussed in the Hadoop community.
-* Hadoop protocols are defined in .proto (ProtocolBuffers) files. Client-Server protocols and Server-protocol .proto files are marked as stable. When a .proto file is marked as stable it means that changes should be made in a compatible fashion as described below:
+* Hadoop protocols are defined in .proto (ProtocolBuffers) files. Client-Server protocols and Server-Server protocol .proto files are marked as stable. When a .proto file is marked as stable it means that changes should be made in a compatible fashion as described below:
     * The following changes are compatible and are allowed at any time:
         * Add an optional field, with the expectation that the code deals with the field missing due to communication with an older version of the code.
         * Add a new rpc/method to the service
@@ -101,7 +101,7 @@ Wire compatibility concerns data being transmitted over the wire between Hadoop
 
 ### Java Binary compatibility for end-user applications i.e. Apache Hadoop ABI
 
-As Apache Hadoop revisions are upgraded end-users reasonably expect that their applications should continue to work without any modifications. This is fulfilled as a result of support API compatibility, Semantic compatibility and Wire compatibility.
+As Apache Hadoop revisions are upgraded end-users reasonably expect that their applications should continue to work without any modifications. This is fulfilled as a result of supporting API compatibility, Semantic compatibility and Wire compatibility.
 
 However, Apache Hadoop is a very complex, distributed system and services a very wide variety of use-cases. In particular, Apache Hadoop MapReduce is a very, very wide API; in the sense that end-users may make wide-ranging assumptions such as layout of the local disk when their map/reduce tasks are executing, environment variables for their tasks etc. In such cases, it becomes very hard to fully specify, and support, absolute compatibility.
 
@@ -115,12 +115,12 @@ However, Apache Hadoop is a very complex, distributed system and services a very
 
 * Existing MapReduce, YARN & HDFS applications and frameworks should work unmodified within a major release i.e. Apache Hadoop ABI is supported.
 * A very minor fraction of applications maybe affected by changes to disk layouts etc., the developer community will strive to minimize these changes and will not make them within a minor version. In more egregious cases, we will consider strongly reverting these breaking changes and invalidating offending releases if necessary.
-* In particular for MapReduce applications, the developer community will try our best to support provide binary compatibility across major releases e.g. applications using org.apache.hadoop.mapred.
+* In particular for MapReduce applications, the developer community will try our best to support providing binary compatibility across major releases e.g. applications using org.apache.hadoop.mapred.
 * APIs are supported compatibly across hadoop-1.x and hadoop-2.x. See [Compatibility for MapReduce applications between hadoop-1.x and hadoop-2.x](../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapReduce_Compatibility_Hadoop1_Hadoop2.html) for more details.
 
 ### REST APIs
 
-REST API compatibility corresponds to both the request (URLs) and responses to each request (content, which may contain other URLs). Hadoop REST APIs are specifically meant for stable use by clients across releases, even major releases. The following are the exposed REST APIs:
+REST API compatibility corresponds to both the requests (URLs) and responses to each request (content, which may contain other URLs). Hadoop REST APIs are specifically meant for stable use by clients across releases, even major ones. The following are the exposed REST APIs:
 
 * [WebHDFS](../hadoop-hdfs/WebHDFS.html) - Stable
 * [ResourceManager](../../hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html)
@@ -136,7 +136,7 @@ The APIs annotated stable in the text above preserve compatibility across at lea
 
 ### Metrics/JMX
 
-While the Metrics API compatibility is governed by Java API compatibility, the actual metrics exposed by Hadoop need to be compatible for users to be able to automate using them (scripts etc.). Adding additional metrics is compatible. Modifying (eg changing the unit or measurement) or removing existing metrics breaks compatibility. Similarly, changes to JMX MBean object names also break compatibility.
+While the Metrics API compatibility is governed by Java API compatibility, the actual metrics exposed by Hadoop need to be compatible for users to be able to automate using them (scripts etc.). Adding additional metrics is compatible. Modifying (e.g. changing the unit or measurement) or removing existing metrics breaks compatibility. Similarly, changes to JMX MBean object names also break compatibility.
 
 #### Policy
 
@@ -148,7 +148,7 @@ User and system level data (including metadata) is stored in files of different
 
 #### User-level file formats
 
-Changes to formats that end-users use to store their data can prevent them for accessing the data in later releases, and hence it is highly important to keep those file-formats compatible. One can always add a "new" format improving upon an existing format. Examples of these formats include har, war, SequenceFileFormat etc.
+Changes to formats that end-users use to store their data can prevent them from accessing the data in later releases, and hence it is highly important to keep those file-formats compatible. One can always add a "new" format improving upon an existing format. Examples of these formats include har, war, SequenceFileFormat etc.
 
 ##### Policy
 
@@ -185,7 +185,7 @@ Depending on the degree of incompatibility in the changes, the following potenti
 
 ### Command Line Interface (CLI)
 
-The Hadoop command line programs may be use either directly via the system shell or via shell scripts. Changing the path of a command, removing or renaming command line options, the order of arguments, or the command return code and output break compatibility and may adversely affect users.
+The Hadoop command line programs may be used either directly via the system shell or via shell scripts. Changing the path of a command, removing or renaming command line options, the order of arguments, or the command return code and output break compatibility and may adversely affect users.
 
 #### Policy
 

hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md

@@ -639,11 +639,11 @@ stat
 
 Usage: `hadoop fs -stat [format] <path> ...`
 
-Print statistics about the file/directory at \<path\> in the specified format. Format accepts filesize in blocks (%b), type (%F), group name of owner (%g), name (%n), block size (%o), replication (%r), user name of owner(%u), and modification date (%y, %Y). %y shows UTC date as "yyyy-MM-dd HH:mm:ss" and %Y shows milliseconds since January 1, 1970 UTC. If the format is not specified, %y is used by default.
+Print statistics about the file/directory at \<path\> in the specified format. Format accepts permissions in octal (%a) and symbolic (%A), filesize in blocks (%b), type (%F), group name of owner (%g), name (%n), block size (%o), replication (%r), user name of owner(%u), and modification date (%y, %Y). %y shows UTC date as "yyyy-MM-dd HH:mm:ss" and %Y shows milliseconds since January 1, 1970 UTC. If the format is not specified, %y is used by default.
 
 Example:
 
-* `hadoop fs -stat "%F %u:%g %b %y %n" /file`
+* `hadoop fs -stat "%F %a %u:%g %b %y %n" /file`
 
 Exit Code: Returns 0 on success and -1 on error.
 

hadoop-common-project/hadoop-common/src/site/markdown/InterfaceClassification.md

@@ -44,7 +44,7 @@ Interfaces have two main attributes: Audience and Stability
 
 Audience denotes the potential consumers of the interface. While many interfaces
 are internal/private to the implementation, other are public/external interfaces
-are meant for wider consumption by applications and/or clients. For example, in
+that are meant for wider consumption by applications and/or clients. For example, in
 posix, libc is an external or public interface, while large parts of the kernel
 are internal or private interfaces. Also, some interfaces are targeted towards
 other specific subsystems.
@@ -52,7 +52,7 @@ other specific subsystems.
 Identifying the audience of an interface helps define the impact of breaking
 it. For instance, it might be okay to break the compatibility of an interface
 whose audience is a small number of specific subsystems. On the other hand, it
-is probably not okay to break a protocol interfaces that millions of Internet
+is probably not okay to break a protocol interface that millions of Internet
 users depend on.
 
 Hadoop uses the following kinds of audience in order of increasing/wider visibility:
@@ -75,7 +75,7 @@ referred to as project-private).
 
 The interface is used by a specified set of projects or systems (typically
 closely related projects). Other projects or systems should not use the
-interface. Changes to the interface will be communicated/ negotiated with the
+interface. Changes to the interface will be communicated/negotiated with the
 specified projects. For example, in the Hadoop project, some interfaces are
 LimitedPrivate{HDFS, MapReduce} in that they are private to the HDFS and
 MapReduce projects.
@@ -92,16 +92,16 @@ the interface are allowed. Hadoop APIs have the following levels of stability.
 #### Stable
 
 Can evolve while retaining compatibility for minor release boundaries; in other
-words, incompatible changes to APIs marked Stable are allowed only at major
+words, incompatible changes to APIs marked as Stable are allowed only at major
 releases (i.e. at m.0).
 
 #### Evolving
 
-Evolving, but incompatible changes are allowed at minor release (i.e. m .x)
+Evolving, but incompatible changes are allowed at minor releases (i.e. m .x)
 
 #### Unstable
 
-Incompatible changes to Unstable APIs are allowed any time. This usually makes
+Incompatible changes to Unstable APIs are allowed at any time. This usually makes
 sense for only private interfaces.
 
 However one may call this out for a supposedly public interface to highlight
@@ -109,11 +109,11 @@ that it should not be used as an interface; for public interfaces, labeling it
 as Not-an-interface is probably more appropriate than "Unstable".
 
 Examples of publicly visible interfaces that are unstable
-(i.e. not-an-interface): GUI, CLIs whose output format will change
+(i.e. not-an-interface): GUI, CLIs whose output format will change.
 
 #### Deprecated
 
-APIs that could potentially removed in the future and should not be used.
+APIs that could potentially be removed in the future and should not be used.
 
 How are the Classifications Recorded?
 -------------------------------------
@@ -155,13 +155,13 @@ FAQ
         * e.g. In HDFS, NN-DN protocol is private but stable and can help
           implement rolling upgrades. It communicates that this interface should
           not be changed in incompatible ways even though it is private.
-        * e.g. In HDFS, FSImage stability can help provide more flexible roll backs.
+        * e.g. In HDFS, FSImage stability provides more flexible rollback.
 
 * What is the harm in applications using a private interface that is stable? How
   is it different than a public stable interface?
     * While a private interface marked as stable is targeted to change only at
       major releases, it may break at other times if the providers of that
-      interface are willing to changes the internal users of that
+      interface are willing to change the internal users of that
       interface. Further, a public stable interface is less likely to break even
       at major releases (even though it is allowed to break compatibility)
       because the impact of the change is larger. If you use a private interface
@@ -182,11 +182,11 @@ FAQ
       away with private then do so; if the interface is really for general use
       for all applications then do so. But remember that making an interface
       public has huge responsibility. Sometimes Limited-private is just right.
-    * A good example of a limited-private interface is BlockLocations, This is
+    * A good example of a limited-private interface is BlockLocations, This is a
       fairly low-level interface that we are willing to expose to MR and perhaps
       HBase. We are likely to change it down the road and at that time we will
-      have get a coordinated effort with the MR team to release matching
-      releases. While MR and HDFS are always released in sync today, they may
+      coordinate release effort with the MR team.
+      While MR and HDFS are always released in sync today, they may
       change down the road.
     * If you have a limited-private interface with many projects listed then you
       are fooling yourself. It is practically public.
@@ -207,7 +207,7 @@ FAQ
       break it at minor releases.
     * One example of a public interface that is unstable is where one is
       providing an implementation of a standards-body based interface that is
-      still under development. For example, many companies, in an attampt to be
+      still under development. For example, many companies, in an attempt to be
       first to market, have provided implementations of a new NFS protocol even
       when the protocol was not fully completed by IETF. The implementor cannot
       evolve the interface in a fashion that causes least distruption because

hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md

@@ -48,7 +48,7 @@ LocalFileSpanReceiver is included in the htrace-core4 jar which is bundled
 with Hadoop.)
 
 ```
-    $ cp htrace-htraced/target/htrace-htraced-4.0.1-incubating.jar $HADOOP_HOME/share/hadoop/common/lib/
+    $ cp htrace-htraced/target/htrace-htraced-4.1.0-incubating.jar $HADOOP_HOME/share/hadoop/common/lib/
 ```
 
 ### Dynamic update of tracing configuration

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md

@@ -35,7 +35,7 @@ of the client.
 
 **Implementation Note**: the static `FileSystem get(URI uri, Configuration conf) ` method MAY return
 a pre-existing instance of a filesystem client class—a class that may also be in use in other threads.
-The implementations of `FileSystem` which ship with Apache Hadoop
+The implementations of `FileSystem` shipped with Apache Hadoop
 *do not make any attempt to synchronize access to the working directory field*.
 
 ## Invariants
@@ -170,6 +170,9 @@ While HDFS currently returns an alphanumerically sorted list, neither the Posix
 nor Java's `File.listFiles()` API calls define any ordering of returned values. Applications
 which require a uniform sort order on the results must perform the sorting themselves.
 
+**Null return**: Local filesystems prior to 3.0.0 returned null upon access
+error. It is considered erroneous. Expect IOException upon access error.
+
 #### Atomicity and Consistency
 
 By the time the `listStatus()` operation returns to the caller, there
@@ -214,7 +217,6 @@ response, then, if a listing `listStatus("/d")` takes place concurrently with th
 
 	[a, part-0000001, ... , part-9999999]
 	[part-0000001, ... , part-9999999, z]
-
 	[a, part-0000001, ... , part-9999999, z]
 	[part-0000001, ... , part-9999999]
 
@@ -282,7 +284,7 @@ value is an instance of the `LocatedFileStatus` subclass of a `FileStatus`,
 and that rather than return an entire list, an iterator is returned.
 
 This is actually a `protected` method, directly invoked by
-`listLocatedStatus(Path path):`. Calls to it may be delegated through
+`listLocatedStatus(Path path)`. Calls to it may be delegated through
 layered filesystems, such as `FilterFileSystem`, so its implementation MUST
 be considered mandatory, even if `listLocatedStatus(Path path)` has been
 implemented in a different manner. There are open JIRAs proposing
@@ -442,7 +444,7 @@ the convention is generally retained.
 
 ###  `long getDefaultBlockSize()`
 
-Get the "default" block size for a filesystem. This often used during
+Get the "default" block size for a filesystem. This is often used during
 split calculations to divide work optimally across a set of worker processes.
 
 #### Preconditions
@@ -604,7 +606,7 @@ This MAY be a bug, as it allows >1 client to create a file with `overwrite==fals
  and potentially confuse file/directory logic
 
 * The Local FileSystem raises a `FileNotFoundException` when trying to create a file over
-a directory, hence it is is listed as an exception that MAY be raised when
+a directory, hence it is listed as an exception that MAY be raised when
 this precondition fails.
 
 * Not covered: symlinks. The resolved path of the symlink is used as the final path argument to the `create()` operation
@@ -898,7 +900,7 @@ Renaming a file where the destination is a directory moves the file as a child
 ##### Renaming a directory onto a directory
 
 If `src` is a directory then all its children will then exist under `dest`, while the path
-`src` and its descendants will no longer not exist. The names of the paths under
+`src` and its descendants will no longer exist. The names of the paths under
 `dest` will match those under `src`, as will the contents:
 
     if isDir(FS, src) isDir(FS, dest) and src != dest :
@@ -928,7 +930,7 @@ The outcome is no change to FileSystem state, with a return value of false.
 *Local Filesystem, S3N*
 
 The outcome is as a normal rename, with the additional (implicit) feature
-that the parent directores of the destination also exist
+that the parent directories of the destination also exist.
 
     exists(FS', parent(dest))
 
@@ -1018,9 +1020,9 @@ HDFS: All source files except the final one MUST be a complete block:
 
 
 HDFS's restrictions may be an implementation detail of how it implements
-`concat` -by changing the inode references to join them together in
+`concat` by changing the inode references to join them together in
 a sequence. As no other filesystem in the Hadoop core codebase
-implements this method, there is no way to distinguish implementation detail.
+implements this method, there is no way to distinguish implementation detail
 from specification.
 
 

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md

@@ -29,7 +29,7 @@ with extensions that add key assumptions to the system.
 1. The stream being read references a finite array of bytes.
 1. The length of the data does not change during the read process.
 1. The contents of the data does not change during the process.
-1. The source file remains present during the read process
+1. The source file remains present during the read process.
 1. Callers may use `Seekable.seek()` to offsets within the array of bytes, with future
 reads starting at this offset.
 1. The cost of forward and backward seeks is low.
@@ -104,7 +104,7 @@ Return the current position. The outcome when a stream is closed is undefined.
 
 Return the data at the current position.
 
-1. Implementations should fail when a stream is closed
+1. Implementations should fail when a stream is closed.
 1. There is no limit on how long `read()` may take to complete.
 
 #### Preconditions
@@ -124,7 +124,7 @@ Return the data at the current position.
 
 Read `length` bytes of data into the destination buffer, starting at offset
 `offset`. The source of the data is the current position of the stream,
-as implicitly set in `pos`
+as implicitly set in `pos`.
 
 #### Preconditions
 
@@ -166,7 +166,7 @@ the stream.
 
 That is, rather than `l` being simply defined as `min(length, len(data)-length)`,
 it strictly is an integer in the range `1..min(length, len(data)-length)`.
-While the caller may expect for as much as the buffer as possible to be filled
+While the caller may expect as much of the buffer as possible to be filled
 in, it is within the specification for an implementation to always return
 a smaller number, perhaps only ever 1 byte.
 
@@ -192,7 +192,7 @@ Some filesystems do not perform this check, relying on the `read()` contract
 to reject reads on a closed stream (e.g. `RawLocalFileSystem`).
 
 A `seek(0)` MUST always succeed, as  the seek position must be
-positive and less than the length of the Stream's:
+positive and less than the length of the Stream:
 
     s > 0 and ((s==0) or ((s < len(data)))) else raise [EOFException, IOException]
 
@@ -222,7 +222,7 @@ data at offset `offset`.
 
 #### Preconditions
 
-Not all subclasses implement the operation operation, and instead
+Not all subclasses implement this operation, and instead
 either raise an exception or return `False`.
 
     supported(FSDIS, Seekable.seekToNewSource) else raise [UnsupportedOperationException, IOException]
@@ -250,7 +250,7 @@ If the operation is supported and there is a new location for the data:
 
 The new data is the original data (or an updated version of it, as covered
 in the Consistency section below), but the block containing the data at `offset`
-sourced from a different replica.
+is sourced from a different replica.
 
 If there is no other copy, `FSDIS` is  not updated; the response indicates this:
 
@@ -258,7 +258,7 @@ If there is no other copy, `FSDIS` is  not updated; the response indicates this:
 
 Outside of test methods, the primary use of this method is in the {{FSInputChecker}}
 class, which can react to a checksum error in a read by attempting to source
-the data elsewhere. It a new source can be found it attempts to reread and
+the data elsewhere. If a new source can be found it attempts to reread and
 recheck that portion of the file.
 
 ## <a name="PositionedReadable"></a> interface `PositionedReadable`

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md

@@ -141,7 +141,7 @@ The failure modes when a user lacks security permissions are not specified.
 
 ### Networking Assumptions
 
-This document assumes this all network operations succeed. All statements
+This document assumes that all network operations succeed. All statements
 can be assumed to be qualified as *"assuming the operation does not fail due
 to a network availability problem"*
 
@@ -303,7 +303,7 @@ does not hold on blob stores]
 1. Directory list operations are fast for directories with few entries, but may
 incur a cost that is `O(entries)`. Hadoop 2 added iterative listing to
 handle the challenge of listing directories with millions of entries without
-buffering -at the cost of consistency.
+buffering at the cost of consistency.
 
 1. A `close()` of an `OutputStream` is fast, irrespective of whether or not
 the file operation has succeeded or not.
@@ -317,8 +317,8 @@ This specification refers to *Object Stores* in places, often using the
 term *Blobstore*. Hadoop does provide FileSystem client classes for some of these
 even though they violate many of the requirements. This is why, although
 Hadoop can read and write data in an object store, the two which Hadoop ships
-with direct support for —Amazon S3 and OpenStack Swift&mdash cannot
-be used as direct replacement for HDFS.
+with direct support for — Amazon S3 and OpenStack Swift — cannot
+be used as direct replacements for HDFS.
 
 *What is an Object Store?*
 
@@ -358,10 +358,10 @@ are current with respect to the files within that directory.
 as are `delete()` operations. Object store FileSystem clients implement these
 as operations on the individual objects whose names match the directory prefix.
 As a result, the changes take place a file at a time, and are not atomic. If
-an operation fails part way through the process, the the state of the object store
+an operation fails part way through the process, then the state of the object store
 reflects the partially completed operation.  Note also that client code
 assumes that these operations are `O(1)` —in an object store they are
-more likely to be be `O(child-entries)`.
+more likely to be `O(child-entries)`.
 
 1. **Durability**. Hadoop assumes that `OutputStream` implementations write data
 to their (persistent) storage on a `flush()` operation. Object store implementations

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/model.md

@@ -18,7 +18,7 @@
 
 ## Paths and Path Elements
 
-A Path is a list of Path elements which represents a path to a file, directory of symbolic link
+A Path is a list of Path elements which represents a path to a file, directory or symbolic link
 
 Path elements are non-empty strings. The exact set of valid strings MAY
 be specific to a particular FileSystem implementation.
@@ -179,7 +179,7 @@ path begins with the path P -that is their parent is P or an ancestor is P
 
 ### File references
 
-A path MAY refer to a file; that it it has data in the filesystem; its path is a key in the data dictionary
+A path MAY refer to a file that has data in the filesystem; its path is a key in the data dictionary
 
     def isFile(FS, p) =  p in FS.Files
 
@@ -206,7 +206,8 @@ process working with the filesystem:
 
 The function `getHomeDirectory` returns the home directory for the Filesystem and the current user account.
 For some FileSystems, the path is `["/","users", System.getProperty("user-name")]`. However,
-for HDFS,
+for HDFS, the username is derived from the credentials used to authenticate the client with HDFS.
+This may differ from the local user account name.
 
 
 ### Exclusivity

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/notation.md

@@ -130,7 +130,7 @@ Strings are lists of characters represented in double quotes. e.g. `"abc"`
 
 All system state declarations are immutable.
 
-The suffix "'" (single quote) is used as the convention to indicate the state of the system after a operation:
+The suffix "'" (single quote) is used as the convention to indicate the state of the system after an operation:
 
     L' = L + ['d','e']
 

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md

@@ -28,7 +28,7 @@ remote server providing the filesystem.
 
 These filesystem bindings must be defined in an XML configuration file, usually
 `hadoop-common-project/hadoop-common/src/test/resources/contract-test-options.xml`.
-This file is excluded should not be checked in.
+This file is excluded and should not be checked in.
 
 ### ftp://
 
@@ -122,7 +122,7 @@ new contract class, then creating a new non-abstract test class for every test
 suite that you wish to test.
 
 1. Do not try and add these tests into Hadoop itself. They won't be added to
-the soutce tree. The tests must live with your own filesystem source.
+the source tree. The tests must live with your own filesystem source.
 1. Create a package in your own test source tree (usually) under `contract`,
 for the files and tests.
 1. Subclass `AbstractFSContract` for your own contract implementation.

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java

@@ -29,6 +29,7 @@ import org.apache.hadoop.fs.local.LocalConfigKeys;
 import org.apache.hadoop.ha.SshFenceByTcpPort;
 import org.apache.hadoop.ha.ZKFailoverController;
 import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
 import org.apache.hadoop.io.nativeio.NativeIO;
 import org.apache.hadoop.security.CompositeGroupsMapping;
 import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
@@ -49,6 +50,7 @@ import org.apache.hadoop.security.ssl.SSLFactory;
  * {@link org.apache.hadoop.security.LdapGroupsMapping}
  * {@link org.apache.hadoop.security.http.CrossOriginFilter}
  * {@link org.apache.hadoop.security.ssl.SSLFactory}
+ * {@link org.apache.hadoop.io.erasurecode.rawcoder.CoderUtil}
  * <p></p>
  * against core-site.xml for missing properties.  Currently only
  * throws an error if the class is missing a property.
@@ -71,7 +73,8 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
         LdapGroupsMapping.class,
         ZKFailoverController.class,
         SSLFactory.class,
-        CompositeGroupsMapping.class
+        CompositeGroupsMapping.class,
+        CodecUtil.class
         };
 
     // Initialize used variables

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSMainOperationsBaseTest.java

@@ -274,8 +274,7 @@ public abstract class FSMainOperationsBaseTest extends FileSystemTestHelper {
       // expected
     }
   }
-  
-  // TODO: update after fixing HADOOP-7352
+
   @Test
   public void testListStatusThrowsExceptionForUnreadableDir()
   throws Exception {
@@ -630,7 +629,26 @@ public abstract class FSMainOperationsBaseTest extends FileSystemTestHelper {
     Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXX),
         filteredPaths));
   }
-  
+
+  @Test
+  public void testGlobStatusThrowsExceptionForUnreadableDir()
+      throws Exception {
+    Path testRootDir = getTestRootPath(fSys, "test/hadoop/dir");
+    Path obscuredDir = new Path(testRootDir, "foo");
+    Path subDir = new Path(obscuredDir, "bar"); //so foo is non-empty
+    fSys.mkdirs(subDir);
+    fSys.setPermission(obscuredDir, new FsPermission((short)0)); //no access
+    try {
+      fSys.globStatus(getTestRootPath(fSys, "test/hadoop/dir/foo/*"));
+      Assert.fail("Should throw IOException");
+    } catch (IOException ioe) {
+      // expected
+    } finally {
+      // make sure the test directory can be deleted
+      fSys.setPermission(obscuredDir, new FsPermission((short)0755)); //default
+    }
+  }
+
   @Test
   public void testWriteReadAndDeleteEmptyFile() throws Exception {
     writeReadAndDelete(0);

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java

@@ -29,13 +29,19 @@ import java.net.URI;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.HashSet;
+import java.util.Random;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import junit.framework.TestCase;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.TrashPolicyDefault.Emptier;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.Time;
+import org.junit.Before;
+import org.junit.Test;
 
 /**
  * This class tests commands from Trash.
@@ -45,6 +51,13 @@ public class TestTrash extends TestCase {
   private final static Path TEST_DIR = new Path(GenericTestUtils.getTempPath(
       "testTrash"));
 
+  @Before
+  public void setUp() throws IOException {
+    // ensure each test initiates a FileSystem instance,
+    // avoid getting an old instance from cache.
+    FileSystem.closeAll();
+  }
+
   protected static Path mkdir(FileSystem fs, Path p) throws IOException {
     assertTrue(fs.mkdirs(p));
     assertTrue(fs.exists(p));
@@ -516,6 +529,81 @@ public class TestTrash extends TestCase {
     assertTrue(trash.getTrashPolicy().getClass().equals(TestTrashPolicy.class));
   }
 
+  @Test
+  public void testCheckpointInterval() throws IOException {
+    // Verify if fs.trash.checkpoint.interval is set to positive number
+    // but bigger than fs.trash.interval,
+    // the value should be reset to fs.trash.interval
+    verifyDefaultPolicyIntervalValues(10, 12, 10);
+
+    // Verify if fs.trash.checkpoint.interval is set to positive number
+    // and smaller than fs.trash.interval, the value should be respected
+    verifyDefaultPolicyIntervalValues(10, 5, 5);
+
+    // Verify if fs.trash.checkpoint.interval sets to 0
+    // the value should be reset to fs.trash.interval
+    verifyDefaultPolicyIntervalValues(10, 0, 10);
+
+    // Verify if fs.trash.checkpoint.interval sets to a negative number
+    // the value should be reset to fs.trash.interval
+    verifyDefaultPolicyIntervalValues(10, -1, 10);
+  }
+
+  @Test
+  public void testMoveEmptyDirToTrash() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setClass(FS_FILE_IMPL_KEY,
+        RawLocalFileSystem.class,
+        FileSystem.class);
+    conf.setLong(FS_TRASH_INTERVAL_KEY, 1); // 1 min
+    FileSystem fs = FileSystem.get(conf);
+    verifyMoveEmptyDirToTrash(fs, conf);
+  }
+
+  /**
+   * Simulate the carrier process of the trash emptier restarts,
+   * verify it honors the <b>fs.trash.interval</b> before and after restart.
+   * @throws Exception
+   */
+  @Test
+  public void testTrashRestarts() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setClass("fs.trash.classname",
+        AuditableTrashPolicy.class,
+        TrashPolicy.class);
+    conf.setClass("fs.file.impl", TestLFS.class, FileSystem.class);
+    conf.set(FS_TRASH_INTERVAL_KEY, "50"); // in milliseconds for test
+    Trash trash = new Trash(conf);
+    // create 5 checkpoints
+    for(int i=0; i<5; i++) {
+      trash.checkpoint();
+    }
+
+    // Run the trash emptier for 120ms, it should run
+    // 2 times deletion as the interval is 50ms.
+    // Verify the checkpoints number when shutting down the emptier.
+    verifyAuditableTrashEmptier(trash, 120, 3);
+
+    // reconfigure the interval to 100 ms
+    conf.set(FS_TRASH_INTERVAL_KEY, "100");
+    Trash trashNew = new Trash(conf);
+
+    // Run the trash emptier for 120ms, it should run
+    // 1 time deletion.
+    verifyAuditableTrashEmptier(trashNew, 120, 2);
+  }
+
+  @Test
+  public void testTrashPermission()  throws IOException {
+    Configuration conf = new Configuration();
+    conf.setClass("fs.trash.classname",
+        TrashPolicyDefault.class,
+        TrashPolicy.class);
+    conf.setClass("fs.file.impl", TestLFS.class, FileSystem.class);
+    conf.set(FS_TRASH_INTERVAL_KEY, "0.2");
+    verifyTrashPermission(FileSystem.getLocal(conf), conf);
+  }
+
   public void testTrashEmptier() throws Exception {
     Configuration conf = new Configuration();
     // Trash with 12 second deletes and 6 seconds checkpoints
@@ -679,12 +767,143 @@ public class TestTrash extends TestCase {
         long factoredTime = first*factor;
         assertTrue(iterTime<factoredTime); //no more then twice of median first 10
       }
-    } 
+    }
   }
-  
-  public static void main(String [] arg) throws IOException{
-    // run performance piece as a separate test
-    performanceTestDeleteSameFile();
+
+  public static void verifyMoveEmptyDirToTrash(FileSystem fs,
+      Configuration conf) throws IOException {
+    Path caseRoot = new Path(
+        GenericTestUtils.getTempPath("testUserTrash"));
+    Path testRoot = new Path(caseRoot, "trash-users");
+    Path emptyDir = new Path(testRoot, "empty-dir");
+    try (FileSystem fileSystem = fs){
+      fileSystem.mkdirs(emptyDir);
+      Trash trash = new Trash(fileSystem, conf);
+      // Make sure trash root is clean
+      Path trashRoot = trash.getCurrentTrashDir(emptyDir);
+      fileSystem.delete(trashRoot, true);
+      // Move to trash should be succeed
+      assertTrue("Move an empty directory to trash failed",
+          trash.moveToTrash(emptyDir));
+      // Verify the empty dir is removed
+      assertFalse("The empty directory still exists on file system",
+          fileSystem.exists(emptyDir));
+      emptyDir = fileSystem.makeQualified(emptyDir);
+      Path dirInTrash = Path.mergePaths(trashRoot, emptyDir);
+      assertTrue("Directory wasn't moved to trash",
+          fileSystem.exists(dirInTrash));
+      FileStatus[] flist = fileSystem.listStatus(dirInTrash);
+      assertTrue("Directory is not empty",
+          flist!= null && flist.length == 0);
+    }
+  }
+
+  /**
+   * Create a bunch of files and set with different permission, after
+   * moved to trash, verify the location in trash directory is expected
+   * and the permission is reserved.
+   *
+   * @throws IOException
+   */
+  public static void verifyTrashPermission(FileSystem fs, Configuration conf)
+      throws IOException {
+    Path caseRoot = new Path(
+        GenericTestUtils.getTempPath("testTrashPermission"));
+    try (FileSystem fileSystem = fs){
+      Trash trash = new Trash(fileSystem, conf);
+      FileSystemTestWrapper wrapper =
+          new FileSystemTestWrapper(fileSystem);
+
+      short[] filePermssions = {
+          (short) 0600,
+          (short) 0644,
+          (short) 0660,
+          (short) 0700,
+          (short) 0750,
+          (short) 0755,
+          (short) 0775,
+          (short) 0777
+      };
+
+      for(int i=0; i<filePermssions.length; i++) {
+        // Set different permission to files
+        FsPermission fsPermission = new FsPermission(filePermssions[i]);
+        Path file = new Path(caseRoot, "file" + i);
+        byte[] randomBytes = new byte[new Random().nextInt(10)];
+        wrapper.writeFile(file, randomBytes);
+        wrapper.setPermission(file, fsPermission);
+
+        // Move file to trash
+        trash.moveToTrash(file);
+
+        // Verify the file is moved to trash, at expected location
+        Path trashDir = trash.getCurrentTrashDir(file);
+        if(!file.isAbsolute()) {
+          file = wrapper.makeQualified(file);
+        }
+        Path fileInTrash = Path.mergePaths(trashDir, file);
+        FileStatus fstat = wrapper.getFileStatus(fileInTrash);
+        assertTrue(String.format("File %s is not moved to trash",
+            fileInTrash.toString()),
+            wrapper.exists(fileInTrash));
+        // Verify permission not change
+        assertTrue(String.format("Expected file: %s is %s, but actual is %s",
+            fileInTrash.toString(),
+            fsPermission.toString(),
+            fstat.getPermission().toString()),
+            fstat.getPermission().equals(fsPermission));
+      }
+
+      // Verify the trash directory can be removed
+      Path trashRoot = trash.getCurrentTrashDir();
+      assertTrue(wrapper.delete(trashRoot, true));
+    }
+  }
+
+  private void verifyDefaultPolicyIntervalValues(long trashInterval,
+      long checkpointInterval, long expectedInterval) throws IOException {
+    Configuration conf = new Configuration();
+    conf.setLong(FS_TRASH_INTERVAL_KEY, trashInterval);
+    conf.set("fs.trash.classname", TrashPolicyDefault.class.getName());
+    conf.setLong(FS_TRASH_CHECKPOINT_INTERVAL_KEY, checkpointInterval);
+    Trash trash = new Trash(conf);
+    Emptier emptier = (Emptier)trash.getEmptier();
+    assertEquals(expectedInterval, emptier.getEmptierInterval());
+  }
+
+  /**
+   * Launch the {@link Trash} emptier for given milliseconds,
+   * verify the number of checkpoints is expected.
+   */
+  private void verifyAuditableTrashEmptier(Trash trash,
+      long timeAlive,
+      int expectedNumOfCheckpoints)
+          throws IOException {
+    Thread emptierThread = null;
+    try {
+      Runnable emptier = trash.getEmptier();
+      emptierThread = new Thread(emptier);
+      emptierThread.start();
+
+      // Shutdown the emptier thread after a given time
+      Thread.sleep(timeAlive);
+      emptierThread.interrupt();
+      emptierThread.join();
+
+      AuditableTrashPolicy at = (AuditableTrashPolicy) trash.getTrashPolicy();
+      assertEquals(
+          String.format("Expected num of checkpoints is %s, but actual is %s",
+              expectedNumOfCheckpoints, at.getNumberOfCheckpoints()),
+          expectedNumOfCheckpoints,
+          at.getNumberOfCheckpoints());
+    } catch (InterruptedException  e) {
+      // Ignore
+    } finally {
+      // Avoid thread leak
+      if(emptierThread != null) {
+        emptierThread.interrupt();
+      }
+    }
   }
 
   // Test TrashPolicy. Don't care about implementation.
@@ -732,4 +951,127 @@ public class TestTrash extends TestCase {
       return null;
     }
   }
+
+  /**
+   * A fake {@link TrashPolicy} implementation, it keeps a count
+   * on number of checkpoints in the trash. It doesn't do anything
+   * other than updating the count.
+   *
+   */
+  public static class AuditableTrashPolicy extends TrashPolicy {
+
+    public AuditableTrashPolicy() {}
+
+    public AuditableTrashPolicy(Configuration conf)
+        throws IOException {
+      this.initialize(conf, null);
+    }
+
+    @Override
+    @Deprecated
+    public void initialize(Configuration conf, FileSystem fs, Path home) {
+      this.deletionInterval = (long)(conf.getFloat(
+          FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT));
+    }
+
+    @Override
+    public void initialize(Configuration conf, FileSystem fs) {
+      this.deletionInterval = (long)(conf.getFloat(
+          FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT));
+    }
+
+    @Override
+    public boolean moveToTrash(Path path) throws IOException {
+      return false;
+    }
+
+    @Override
+    public void createCheckpoint() throws IOException {
+      AuditableCheckpoints.add();
+    }
+
+    @Override
+    public void deleteCheckpoint() throws IOException {
+      AuditableCheckpoints.delete();
+    }
+
+    @Override
+    public Path getCurrentTrashDir() {
+      return null;
+    }
+
+    @Override
+    public Runnable getEmptier() throws IOException {
+      return new AuditableEmptier(getConf());
+    }
+
+    public int getNumberOfCheckpoints() {
+      return AuditableCheckpoints.get();
+    }
+
+    /**
+     * A fake emptier that simulates to delete a checkpoint
+     * in a fixed interval.
+     */
+    private class AuditableEmptier implements Runnable {
+      private Configuration conf = null;
+      public AuditableEmptier(Configuration conf) {
+        this.conf = conf;
+      }
+
+      @Override
+      public void run() {
+        AuditableTrashPolicy trash = null;
+        try {
+          trash = new AuditableTrashPolicy(conf);
+        } catch (IOException e1) {}
+        while(true) {
+          try {
+            Thread.sleep(deletionInterval);
+            trash.deleteCheckpoint();
+          } catch (IOException e) {
+            // no exception
+          } catch (InterruptedException e) {
+            break;
+          }
+        }
+      }
+    }
+
+    @Override
+    public boolean isEnabled() {
+      return true;
+    }
+  }
+
+  /**
+   * Only counts the number of checkpoints, not do anything more.
+   * Declared as an inner static class to share state between
+   * testing threads.
+   */
+  private static class AuditableCheckpoints {
+
+    private static AtomicInteger numOfCheckpoint =
+        new AtomicInteger(0);
+
+    private static void add() {
+      numOfCheckpoint.incrementAndGet();
+      System.out.println(String
+          .format("Create a checkpoint, current number of checkpoints %d",
+              numOfCheckpoint.get()));
+    }
+
+    private static void delete() {
+      if(numOfCheckpoint.get() > 0) {
+        numOfCheckpoint.decrementAndGet();
+        System.out.println(String
+            .format("Delete a checkpoint, current number of checkpoints %d",
+                numOfCheckpoint.get()));
+      }
+    }
+
+    private static int get() {
+      return numOfCheckpoint.get();
+    }
+  }
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRootDirectoryTest.java

@@ -27,12 +27,16 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.test.LambdaTestUtils;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.deleteChildren;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dumpStats;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.listChildren;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.toList;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.treeWalk;
@@ -45,6 +49,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.treeWalk;
 public abstract class AbstractContractRootDirectoryTest extends AbstractFSContractTestBase {
   private static final Logger LOG =
       LoggerFactory.getLogger(AbstractContractRootDirectoryTest.class);
+  public static final int OBJECTSTORE_RETRY_TIMEOUT = 30000;
 
   @Override
   public void setup() throws Exception {
@@ -79,23 +84,34 @@ public abstract class AbstractContractRootDirectoryTest extends AbstractFSContra
     // extra sanity checks here to avoid support calls about complete loss
     // of data
     skipIfUnsupported(TEST_ROOT_TESTS_ENABLED);
-    Path root = new Path("/");
+    final Path root = new Path("/");
     assertIsDirectory(root);
-    // make sure it is clean
-    FileSystem fs = getFileSystem();
-    deleteChildren(fs, root, true);
-    FileStatus[] children = listChildren(fs, root);
-    if (children.length > 0) {
-      StringBuilder error = new StringBuilder();
-      error.append("Deletion of child entries failed, still have")
-          .append(children.length)
-          .append(System.lineSeparator());
-      for (FileStatus child : children) {
-        error.append("  ").append(child.getPath())
-            .append(System.lineSeparator());
-      }
-      fail(error.toString());
-    }
+    // make sure the directory is clean. This includes some retry logic
+    // to forgive blobstores whose listings can be out of sync with the file
+    // status;
+    final FileSystem fs = getFileSystem();
+    final AtomicInteger iterations = new AtomicInteger(0);
+    final FileStatus[] originalChildren = listChildren(fs, root);
+    LambdaTestUtils.eventually(
+        OBJECTSTORE_RETRY_TIMEOUT,
+        new Callable<Void>() {
+          @Override
+          public Void call() throws Exception {
+            FileStatus[] deleted = deleteChildren(fs, root, true);
+            FileStatus[] children = listChildren(fs, root);
+            if (children.length > 0) {
+              fail(String.format(
+                  "After %d attempts: listing after rm /* not empty"
+                      + "\n%s\n%s\n%s",
+                  iterations.incrementAndGet(),
+                  dumpStats("final", children),
+                  dumpStats("deleted", deleted),
+                  dumpStats("original", originalChildren)));
+            }
+            return null;
+          }
+        },
+        new LambdaTestUtils.ProportionalRetryInterval(50, 1000));
     // then try to delete the empty one
     boolean deleted = fs.delete(root, false);
     LOG.info("rm / of empty dir result is {}", deleted);

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java

@@ -400,18 +400,18 @@ public class ContractTestUtils extends Assert {
    * @param fileSystem filesystem
    * @param path path to delete
    * @param recursive flag to indicate child entry deletion should be recursive
-   * @return the number of child entries found and deleted (not including
+   * @return the immediate child entries found and deleted (not including
    * any recursive children of those entries)
    * @throws IOException problem in the deletion process.
    */
-  public static int deleteChildren(FileSystem fileSystem,
+  public static FileStatus[] deleteChildren(FileSystem fileSystem,
       Path path,
       boolean recursive) throws IOException {
     FileStatus[] children = listChildren(fileSystem, path);
     for (FileStatus entry : children) {
       fileSystem.delete(entry.getPath(), recursive);
     }
-    return children.length;
+    return children;
   }
 
   /**
@@ -965,7 +965,7 @@ public class ContractTestUtils extends Assert {
    * @return the number of megabytes/second of the recorded operation
    */
   public static double bandwidthMBs(long bytes, long durationNS) {
-    return (bytes * 1000.0) / durationNS;
+    return bytes / (1024.0 * 1024) * 1.0e9 / durationNS;
   }
 
   /**
@@ -1415,6 +1415,14 @@ public class ContractTestUtils extends Assert {
       return endTime - startTime;
     }
 
+    /**
+     * Intermediate duration of the operation.
+     * @return how much time has passed since the start (in nanos).
+     */
+    public long elapsedTime() {
+      return now() - startTime;
+    }
+
     public double bandwidth(long bytes) {
       return bandwidthMBs(bytes, duration());
     }
@@ -1422,10 +1430,12 @@ public class ContractTestUtils extends Assert {
     /**
      * Bandwidth as bytes per second.
      * @param bytes bytes in
-     * @return the number of bytes per second this operation timed.
+     * @return the number of bytes per second this operation.
+     *         0 if duration == 0.
      */
     public double bandwidthBytes(long bytes) {
-      return (bytes * 1.0) / duration();
+      double duration = duration();
+      return duration > 0 ? bytes / duration : 0;
     }
 
     /**

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java

@@ -30,9 +30,11 @@ import java.util.Arrays;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.Shell;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -217,6 +219,23 @@ public class TestPathData {
     );
   }
 
+  @Test
+  public void testGlobThrowsExceptionForUnreadableDir() throws Exception {
+    Path obscuredDir = new Path("foo");
+    Path subDir = new Path(obscuredDir, "bar"); //so foo is non-empty
+    fs.mkdirs(subDir);
+    fs.setPermission(obscuredDir, new FsPermission((short)0)); //no access
+    try {
+      PathData.expandAsGlob("foo/*", conf);
+      Assert.fail("Should throw IOException");
+    } catch (IOException ioe) {
+      // expected
+    } finally {
+      // make sure the test directory can be deleted
+      fs.setPermission(obscuredDir, new FsPermission((short)0755)); //default
+    }
+  }
+
   @Test (timeout = 30000)
   public void testWithStringAndConfForBuggyPath() throws Exception {
     String dirString = "file:///tmp";

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsConfig.java

@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -22,49 +22,41 @@ import java.net.URI;
 import java.net.URISyntaxException;
 
 import org.apache.hadoop.conf.Configuration;
-
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.fs.viewfs.ConfigUtil;
-import org.apache.hadoop.fs.viewfs.InodeTree;
 import org.junit.Test;
 
-
 public class TestViewFsConfig {
-  
-  
-  @Test(expected=FileAlreadyExistsException.class)
+
+  @Test(expected = FileAlreadyExistsException.class)
   public void testInvalidConfig() throws IOException, URISyntaxException {
     Configuration conf = new Configuration();
     ConfigUtil.addLink(conf, "/internalDir/linkToDir2",
         new Path("file:///dir2").toUri());
     ConfigUtil.addLink(conf, "/internalDir/linkToDir2/linkToDir3",
         new Path("file:///dir3").toUri());
-    
-    class Foo { };
-    
-     new InodeTree<Foo>(conf, null) {
+
+    class Foo {
+    }
+
+    new InodeTree<Foo>(conf, null) {
 
       @Override
-      protected
-      Foo getTargetFileSystem(final URI uri)
-        throws URISyntaxException, UnsupportedFileSystemException {
-          return null;
+      protected Foo getTargetFileSystem(final URI uri)
+          throws URISyntaxException, UnsupportedFileSystemException {
+        return null;
       }
 
       @Override
-      protected
-      Foo getTargetFileSystem(
-          org.apache.hadoop.fs.viewfs.InodeTree.INodeDir<Foo>
-                                          dir)
-        throws URISyntaxException {
+      protected Foo getTargetFileSystem(
+          org.apache.hadoop.fs.viewfs.InodeTree.INodeDir<Foo> dir)
+          throws URISyntaxException {
         return null;
       }
 
       @Override
-      protected
-      Foo getTargetFileSystem(URI[] mergeFsURIList)
+      protected Foo getTargetFileSystem(URI[] mergeFsURIList)
           throws URISyntaxException, UnsupportedFileSystemException {
         return null;
       }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java

@@ -18,7 +18,6 @@
 package org.apache.hadoop.io.erasurecode;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder;
 import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoderLegacy;
 import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder;
@@ -73,7 +72,7 @@ public class TestCodecRawCoderMapping {
     String dummyFactName = "DummyNoneExistingFactory";
     // set the dummy factory to rs-legacy and create a raw coder
     // with rs-default, which is OK as the raw coder key is not used
-    conf.set(CommonConfigurationKeys.
+    conf.set(CodecUtil.
         IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, dummyFactName);
     RawErasureEncoder encoder = CodecUtil.createRawEncoder(conf,
         ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/codec/TestHHXORErasureCodec.java

@@ -17,7 +17,9 @@
  */
 package org.apache.hadoop.io.erasurecode.codec;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.ErasureCodecOptions;
 import org.apache.hadoop.io.erasurecode.coder.ErasureCoder;
 import org.junit.Test;
 
@@ -25,10 +27,12 @@ import static org.junit.Assert.assertEquals;
 
 public class TestHHXORErasureCodec {
   private ECSchema schema = new ECSchema("hhxor", 10, 4);
+  private ErasureCodecOptions options = new ErasureCodecOptions(schema);
 
   @Test
   public void testGoodCodec() {
-    HHXORErasureCodec codec = new HHXORErasureCodec(schema);
+    HHXORErasureCodec codec
+        = new HHXORErasureCodec(new Configuration(), options);
     ErasureCoder encoder = codec.createEncoder();
     assertEquals(10, encoder.getNumDataUnits());
     assertEquals(4, encoder.getNumParityUnits());

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.io.erasurecode.coder;
 import org.apache.hadoop.io.erasurecode.ECBlock;
 import org.apache.hadoop.io.erasurecode.ECBlockGroup;
 import org.apache.hadoop.io.erasurecode.ECChunk;
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 import org.apache.hadoop.io.erasurecode.TestCoderBase;
 
 import java.lang.reflect.Constructor;
@@ -158,10 +159,12 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
   protected ErasureCoder createEncoder() {
     ErasureCoder encoder;
     try {
+      ErasureCoderOptions options = new ErasureCoderOptions(
+          numDataUnits, numParityUnits, allowChangeInputs, allowDump);
       Constructor<? extends ErasureCoder> constructor =
           (Constructor<? extends ErasureCoder>)
-              encoderClass.getConstructor(int.class, int.class);
-      encoder = constructor.newInstance(numDataUnits, numParityUnits);
+              encoderClass.getConstructor(ErasureCoderOptions.class);
+      encoder = constructor.newInstance(options);
     } catch (Exception e) {
       throw new RuntimeException("Failed to create encoder", e);
     }
@@ -177,10 +180,12 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
   protected ErasureCoder createDecoder() {
     ErasureCoder decoder;
     try {
+      ErasureCoderOptions options = new ErasureCoderOptions(
+          numDataUnits, numParityUnits, allowChangeInputs, allowDump);
       Constructor<? extends ErasureCoder> constructor =
           (Constructor<? extends ErasureCoder>)
-              decoderClass.getConstructor(int.class, int.class);
-      decoder = constructor.newInstance(numDataUnits, numParityUnits);
+              decoderClass.getConstructor(ErasureCoderOptions.class);
+      decoder = constructor.newInstance(options);
     } catch (Exception e) {
       throw new RuntimeException("Failed to create decoder", e);
     }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java

@@ -18,7 +18,7 @@
 package org.apache.hadoop.io.erasurecode.coder;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
 import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
 import org.junit.Before;
 import org.junit.Test;
@@ -50,7 +50,7 @@ public class TestHHXORErasureCoder extends TestHHErasureCoderBase {
      * This tests if the configuration items work or not.
      */
     Configuration conf = new Configuration();
-    conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+    conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
         RSRawErasureCoderFactory.class.getCanonicalName());
     prepare(conf, 10, 4, new int[]{0}, new int[0]);
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java

@@ -18,7 +18,7 @@
 package org.apache.hadoop.io.erasurecode.coder;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
 import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
 import org.junit.Before;
 import org.junit.Rule;
@@ -57,7 +57,7 @@ public class TestRSErasureCoder extends TestErasureCoderBase {
      * This tests if the configuration items work or not.
      */
     Configuration conf = new Configuration();
-    conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+    conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
         RSRawErasureCoderFactory.class.getCanonicalName());
     prepare(conf, 10, 4, new int[]{0}, new int[0]);
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/LambdaTestUtils.java

@@ -0,0 +1,521 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.test;
+
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.util.Time;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Class containing methods and associated classes to make the most of Lambda
+ * expressions in Hadoop tests.
+ *
+ * The code has been designed from the outset to be Java-8 friendly, but
+ * to still be usable in Java 7.
+ *
+ * The code is modelled on {@code GenericTestUtils#waitFor(Supplier, int, int)},
+ * but also lifts concepts from Scalatest's {@code awaitResult} and
+ * its notion of pluggable retry logic (simple, backoff, maybe even things
+ * with jitter: test author gets to choose).
+ * The {@link #intercept(Class, Callable)} method is also all credit due
+ * Scalatest, though it's been extended to also support a string message
+ * check; useful when checking the contents of the exception.
+ */
+public final class LambdaTestUtils {
+  public static final Logger LOG =
+      LoggerFactory.getLogger(LambdaTestUtils.class);
+
+  private LambdaTestUtils() {
+  }
+
+  /**
+   * This is the string included in the assertion text in
+   * {@link #intercept(Class, Callable)} if
+   * the closure returned a null value.
+   */
+  public static final String NULL_RESULT = "(null)";
+
+  /**
+   * Interface to implement for converting a timeout into some form
+   * of exception to raise.
+   */
+  public interface TimeoutHandler {
+
+    /**
+     * Create an exception (or throw one, if desired).
+     * @param timeoutMillis timeout which has arisen
+     * @param caught any exception which was caught; may be null
+     * @return an exception which will then be thrown
+     * @throws Exception if the handler wishes to raise an exception
+     * that way.
+     */
+    Exception evaluate(int timeoutMillis, Exception caught) throws Exception;
+  }
+
+  /**
+   * Wait for a condition to be met, with a retry policy returning the
+   * sleep time before the next attempt is made. If, at the end
+   * of the timeout period, the condition is still false (or failing with
+   * an exception), the timeout handler is invoked, passing in the timeout
+   * and any exception raised in the last invocation. The exception returned
+   * by this timeout handler is then rethrown.
+   * <p>
+   * Example: Wait 30s for a condition to be met, with a sleep of 30s
+   * between each probe.
+   * If the operation is failing, then, after 30s, the timeout handler
+   * is called. This returns the exception passed in (if any),
+   * or generates a new one.
+   * <pre>
+   * await(
+   *   30 * 1000,
+   *   () -> { return 0 == filesystem.listFiles(new Path("/")).length); },
+   *   () -> 500),
+   *   (timeout, ex) -> ex != null ? ex : new TimeoutException("timeout"));
+   * </pre>
+   *
+   * @param timeoutMillis timeout in milliseconds.
+   * Can be zero, in which case only one attempt is made.
+   * @param check predicate to evaluate
+   * @param retry retry escalation logic
+   * @param timeoutHandler handler invoked on timeout;
+   * the returned exception will be thrown
+   * @return the number of iterations before the condition was satisfied
+   * @throws Exception the exception returned by {@code timeoutHandler} on
+   * timeout
+   * @throws FailFastException immediately if the evaluated operation raises it
+   * @throws InterruptedException if interrupted.
+   */
+  public static int await(int timeoutMillis,
+      Callable<Boolean> check,
+      Callable<Integer> retry,
+      TimeoutHandler timeoutHandler)
+      throws Exception {
+    Preconditions.checkArgument(timeoutMillis >= 0,
+        "timeoutMillis must be >= 0");
+    Preconditions.checkNotNull(timeoutHandler);
+
+    long endTime = Time.now() + timeoutMillis;
+    Exception ex = null;
+    boolean running = true;
+    int iterations = 0;
+    while (running) {
+      iterations++;
+      try {
+        if (check.call()) {
+          return iterations;
+        }
+        // the probe failed but did not raise an exception. Reset any
+        // exception raised by a previous probe failure.
+        ex = null;
+      } catch (InterruptedException | FailFastException e) {
+        throw e;
+      } catch (Exception e) {
+        LOG.debug("eventually() iteration {}", iterations, e);
+        ex = e;
+      }
+      running = Time.now() < endTime;
+      if (running) {
+        int sleeptime = retry.call();
+        if (sleeptime >= 0) {
+          Thread.sleep(sleeptime);
+        } else {
+          running = false;
+        }
+      }
+    }
+    // timeout
+    Exception evaluate = timeoutHandler.evaluate(timeoutMillis, ex);
+    if (evaluate == null) {
+      // bad timeout handler logic; fall back to GenerateTimeout so the
+      // underlying problem isn't lost.
+      LOG.error("timeout handler {} did not throw an exception ",
+          timeoutHandler);
+      evaluate = new GenerateTimeout().evaluate(timeoutMillis, ex);
+    }
+    throw evaluate;
+  }
+
+  /**
+   * Simplified {@link #await(int, Callable, Callable, TimeoutHandler)}
+   * operation with a fixed interval
+   * and {@link GenerateTimeout} handler to generate a {@code TimeoutException}.
+   * <p>
+   * Example: await for probe to succeed:
+   * <pre>
+   * await(
+   *   30 * 1000, 500,
+   *   () -> { return 0 == filesystem.listFiles(new Path("/")).length); });
+   * </pre>
+   *
+   * @param timeoutMillis timeout in milliseconds.
+   * Can be zero, in which case only one attempt is made.
+   * @param intervalMillis interval in milliseconds between checks
+   * @param check predicate to evaluate
+   * @return the number of iterations before the condition was satisfied
+   * @throws Exception returned by {@code failure} on timeout
+   * @throws FailFastException immediately if the evaluated operation raises it
+   * @throws InterruptedException if interrupted.
+   */
+  public static int await(int timeoutMillis,
+      int intervalMillis,
+      Callable<Boolean> check) throws Exception {
+    return await(timeoutMillis, check,
+        new FixedRetryInterval(intervalMillis),
+        new GenerateTimeout());
+  }
+
+  /**
+   * Repeatedly execute a closure until it returns a value rather than
+   * raise an exception.
+   * Exceptions are caught and, with one exception,
+   * trigger a sleep and retry. This is similar of ScalaTest's
+   * {@code eventually(timeout, closure)} operation, though that lacks
+   * the ability to fail fast if the inner closure has determined that
+   * a failure condition is non-recoverable.
+   * <p>
+   * Example: spin until an the number of files in a filesystem is non-zero,
+   * returning the files found.
+   * The sleep interval backs off by 500 ms each iteration to a maximum of 5s.
+   * <pre>
+   * FileStatus[] files = eventually( 30 * 1000,
+   *   () -> {
+   *     FileStatus[] f = filesystem.listFiles(new Path("/"));
+   *     assertEquals(0, f.length);
+   *     return f;
+   *   },
+   *   new ProportionalRetryInterval(500, 5000));
+   * </pre>
+   * This allows for a fast exit, yet reduces probe frequency over time.
+   *
+   * @param <T> return type
+   * @param timeoutMillis timeout in milliseconds.
+   * Can be zero, in which case only one attempt is made before failing.
+   * @param eval expression to evaluate
+   * @param retry retry interval generator
+   * @return result of the first successful eval call
+   * @throws Exception the last exception thrown before timeout was triggered
+   * @throws FailFastException if raised -without any retry attempt.
+   * @throws InterruptedException if interrupted during the sleep operation.
+   */
+  public static <T> T eventually(int timeoutMillis,
+      Callable<T> eval,
+      Callable<Integer> retry) throws Exception {
+    Preconditions.checkArgument(timeoutMillis >= 0,
+        "timeoutMillis must be >= 0");
+    long endTime = Time.now() + timeoutMillis;
+    Exception ex;
+    boolean running;
+    int sleeptime;
+    int iterations = 0;
+    do {
+      iterations++;
+      try {
+        return eval.call();
+      } catch (InterruptedException | FailFastException e) {
+        // these two exceptions trigger an immediate exit
+        throw e;
+      } catch (Exception e) {
+        LOG.debug("evaluate() iteration {}", iterations, e);
+        ex = e;
+      }
+      running = Time.now() < endTime;
+      if (running && (sleeptime = retry.call()) >= 0) {
+        Thread.sleep(sleeptime);
+      }
+    } while (running);
+    // timeout. Throw the last exception raised
+    throw ex;
+  }
+
+  /**
+   * Simplified {@link #eventually(int, Callable, Callable)} method
+   * with a fixed interval.
+   * <p>
+   * Example: wait 30s until an assertion holds, sleeping 1s between each
+   * check.
+   * <pre>
+   * eventually( 30 * 1000, 1000,
+   *   () -> { assertEquals(0, filesystem.listFiles(new Path("/")).length); }
+   * );
+   * </pre>
+   *
+   * @param timeoutMillis timeout in milliseconds.
+   * Can be zero, in which case only one attempt is made before failing.
+   * @param intervalMillis interval in milliseconds
+   * @param eval expression to evaluate
+   * @return result of the first successful invocation of {@code eval()}
+   * @throws Exception the last exception thrown before timeout was triggered
+   * @throws FailFastException if raised -without any retry attempt.
+   * @throws InterruptedException if interrupted during the sleep operation.
+   */
+  public static <T> T eventually(int timeoutMillis,
+      int intervalMillis,
+      Callable<T> eval) throws Exception {
+    return eventually(timeoutMillis, eval,
+        new FixedRetryInterval(intervalMillis));
+  }
+
+  /**
+   * Intercept an exception; throw an {@code AssertionError} if one not raised.
+   * The caught exception is rethrown if it is of the wrong class or
+   * does not contain the text defined in {@code contained}.
+   * <p>
+   * Example: expect deleting a nonexistent file to raise a
+   * {@code FileNotFoundException}.
+   * <pre>
+   * FileNotFoundException ioe = intercept(FileNotFoundException.class,
+   *   () -> {
+   *     filesystem.delete(new Path("/missing"), false);
+   *   });
+   * </pre>
+   *
+   * @param clazz class of exception; the raised exception must be this class
+   * <i>or a subclass</i>.
+   * @param eval expression to eval
+   * @param <T> return type of expression
+   * @param <E> exception class
+   * @return the caught exception if it was of the expected type
+   * @throws Exception any other exception raised
+   * @throws AssertionError if the evaluation call didn't raise an exception.
+   * The error includes the {@code toString()} value of the result, if this
+   * can be determined.
+   */
+  @SuppressWarnings("unchecked")
+  public static <T, E extends Throwable> E intercept(
+      Class<E> clazz,
+      Callable<T> eval)
+      throws Exception {
+    try {
+      T result = eval.call();
+      throw new AssertionError("Expected an exception, got "
+          + robustToString(result));
+    } catch (Throwable e) {
+      if (clazz.isAssignableFrom(e.getClass())) {
+        return (E)e;
+      }
+      throw e;
+    }
+  }
+
+  /**
+   * Intercept an exception; throw an {@code AssertionError} if one not raised.
+   * The caught exception is rethrown if it is of the wrong class or
+   * does not contain the text defined in {@code contained}.
+   * <p>
+   * Example: expect deleting a nonexistent file to raise a
+   * {@code FileNotFoundException} with the {@code toString()} value
+   * containing the text {@code "missing"}.
+   * <pre>
+   * FileNotFoundException ioe = intercept(FileNotFoundException.class,
+   *   "missing",
+   *   () -> {
+   *     filesystem.delete(new Path("/missing"), false);
+   *   });
+   * </pre>
+   *
+   * @param clazz class of exception; the raised exception must be this class
+   * <i>or a subclass</i>.
+   * @param contained string which must be in the {@code toString()} value
+   * of the exception
+   * @param eval expression to eval
+   * @param <T> return type of expression
+   * @param <E> exception class
+   * @return the caught exception if it was of the expected type and contents
+   * @throws Exception any other exception raised
+   * @throws AssertionError if the evaluation call didn't raise an exception.
+   * The error includes the {@code toString()} value of the result, if this
+   * can be determined.
+   * @see GenericTestUtils#assertExceptionContains(String, Throwable)
+   */
+  public static <T, E extends Throwable> E intercept(
+      Class<E> clazz,
+      String contained,
+      Callable<T> eval)
+      throws Exception {
+    E ex = intercept(clazz, eval);
+    GenericTestUtils.assertExceptionContains(contained, ex);
+    return ex;
+  }
+
+  /**
+   * Robust string converter for exception messages; if the {@code toString()}
+   * method throws an exception then that exception is caught and logged,
+   * then a simple string of the classname logged.
+   * This stops a {@code toString()} failure hiding underlying problems.
+   * @param o object to stringify
+   * @return a string for exception messages
+   */
+  private static String robustToString(Object o) {
+    if (o == null) {
+      return NULL_RESULT;
+    } else {
+      try {
+        return o.toString();
+      } catch (Exception e) {
+        LOG.info("Exception calling toString()", e);
+        return o.getClass().toString();
+      }
+    }
+  }
+
+  /**
+   * Returns {@code TimeoutException} on a timeout. If
+   * there was a inner class passed in, includes it as the
+   * inner failure.
+   */
+  public static class GenerateTimeout implements TimeoutHandler {
+    private final String message;
+
+    public GenerateTimeout(String message) {
+      this.message = message;
+    }
+
+    public GenerateTimeout() {
+      this("timeout");
+    }
+
+    /**
+     * Evaluate operation creates a new {@code TimeoutException}.
+     * @param timeoutMillis timeout in millis
+     * @param caught optional caught exception
+     * @return TimeoutException
+     */
+    @Override
+    public Exception evaluate(int timeoutMillis, Exception caught)
+        throws Exception {
+      String s = String.format("%s: after %d millis", message,
+          timeoutMillis);
+      String caughtText = caught != null
+          ? ("; " + robustToString(caught)) : "";
+
+      return (TimeoutException) (new TimeoutException(s + caughtText)
+                                     .initCause(caught));
+    }
+  }
+
+  /**
+   * Retry at a fixed time period between calls.
+   */
+  public static class FixedRetryInterval implements Callable<Integer> {
+    private final int intervalMillis;
+    private int invocationCount = 0;
+
+    public FixedRetryInterval(int intervalMillis) {
+      Preconditions.checkArgument(intervalMillis > 0);
+      this.intervalMillis = intervalMillis;
+    }
+
+    @Override
+    public Integer call() throws Exception {
+      invocationCount++;
+      return intervalMillis;
+    }
+
+    public int getInvocationCount() {
+      return invocationCount;
+    }
+
+    @Override
+    public String toString() {
+      final StringBuilder sb = new StringBuilder(
+          "FixedRetryInterval{");
+      sb.append("interval=").append(intervalMillis);
+      sb.append(", invocationCount=").append(invocationCount);
+      sb.append('}');
+      return sb.toString();
+    }
+  }
+
+  /**
+   * Gradually increase the sleep time by the initial interval, until
+   * the limit set by {@code maxIntervalMillis} is reached.
+   */
+  public static class ProportionalRetryInterval implements Callable<Integer> {
+    private final int intervalMillis;
+    private final int maxIntervalMillis;
+    private int current;
+    private int invocationCount = 0;
+
+    public ProportionalRetryInterval(int intervalMillis,
+        int maxIntervalMillis) {
+      Preconditions.checkArgument(intervalMillis > 0);
+      Preconditions.checkArgument(maxIntervalMillis > 0);
+      this.intervalMillis = intervalMillis;
+      this.current = intervalMillis;
+      this.maxIntervalMillis = maxIntervalMillis;
+    }
+
+    @Override
+    public Integer call() throws Exception {
+      invocationCount++;
+      int last = current;
+      if (last < maxIntervalMillis) {
+        current += intervalMillis;
+      }
+      return last;
+    }
+
+    public int getInvocationCount() {
+      return invocationCount;
+    }
+
+    @Override
+    public String toString() {
+      final StringBuilder sb = new StringBuilder(
+          "ProportionalRetryInterval{");
+      sb.append("interval=").append(intervalMillis);
+      sb.append(", current=").append(current);
+      sb.append(", limit=").append(maxIntervalMillis);
+      sb.append(", invocationCount=").append(invocationCount);
+      sb.append('}');
+      return sb.toString();
+    }
+  }
+
+  /**
+   * An exception which triggers a fast exist from the
+   * {@link #eventually(int, Callable, Callable)} and
+   * {@link #await(int, Callable, Callable, TimeoutHandler)} loops.
+   */
+  public static class FailFastException extends Exception {
+
+    public FailFastException(String detailMessage) {
+      super(detailMessage);
+    }
+
+    public FailFastException(String message, Throwable cause) {
+      super(message, cause);
+    }
+
+    /**
+     * Instantiate from a format string.
+     * @param format format string
+     * @param args arguments to format
+     * @return an instance with the message string constructed.
+     */
+    public static FailFastException newInstance(String format, Object...args) {
+      return new FailFastException(String.format(format, args));
+    }
+  }
+}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestLambdaTestUtils.java

@@ -0,0 +1,395 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.test;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.hadoop.test.LambdaTestUtils.*;
+import static org.apache.hadoop.test.GenericTestUtils.*;
+
+/**
+ * Test the logic in {@link LambdaTestUtils}.
+ * This test suite includes Java 8 and Java 7 code; the Java 8 code exists
+ * to verify that the API is easily used with Lambda expressions.
+ */
+public class TestLambdaTestUtils extends Assert {
+
+  public static final int INTERVAL = 10;
+  public static final int TIMEOUT = 50;
+  private FixedRetryInterval retry = new FixedRetryInterval(INTERVAL);
+  // counter for lambda expressions to use
+  private int count;
+
+  /**
+   * Always evaluates to true.
+   */
+  public static final Callable<Boolean> ALWAYS_TRUE =
+      new Callable<Boolean>() {
+        @Override
+        public Boolean call() throws Exception {
+          return true;
+        }
+      };
+
+  /**
+   * Always evaluates to false.
+   */
+  public static final Callable<Boolean> ALWAYS_FALSE =
+      new Callable<Boolean>() {
+        @Override
+        public Boolean call() throws Exception {
+          return false;
+        }
+      };
+
+  /**
+   * Text in the raised FNFE.
+   */
+  public static final String MISSING = "not found";
+
+  /**
+   * A predicate that always throws a FileNotFoundException.
+   */
+  public static final Callable<Boolean> ALWAYS_FNFE =
+      new Callable<Boolean>() {
+        @Override
+        public Boolean call() throws Exception {
+          throw new FileNotFoundException(MISSING);
+        }
+      };
+
+  /**
+   * reusable timeout handler.
+   */
+  public static final GenerateTimeout
+      TIMEOUT_FAILURE_HANDLER = new GenerateTimeout();
+
+  /**
+   * Always evaluates to 3L.
+   */
+  public static final Callable<Long> EVAL_3L = new Callable<Long>() {
+    @Override
+    public Long call() throws Exception {
+      return 3L;
+    }
+  };
+
+  /**
+   * Always raises a {@code FileNotFoundException}.
+   */
+  public static final Callable<Long> EVAL_FNFE = new Callable<Long>() {
+    @Override
+    public Long call() throws Exception {
+      throw new FileNotFoundException(MISSING);
+    }
+  };
+
+  /**
+   * Assert the retry count is as expected.
+   * @param expected expected value
+   */
+  protected void assertRetryCount(int expected) {
+    assertEquals(retry.toString(), expected, retry.getInvocationCount());
+  }
+
+  /**
+   * Assert the retry count is as expected.
+   * @param minCount minimum value
+   */
+  protected void assertMinRetryCount(int minCount) {
+    assertTrue("retry count of " + retry + " is not >= " + minCount,
+        minCount <= retry.getInvocationCount());
+  }
+
+  @Test
+  public void testAwaitAlwaysTrue() throws Throwable {
+    await(TIMEOUT,
+        ALWAYS_TRUE,
+        new FixedRetryInterval(INTERVAL),
+        TIMEOUT_FAILURE_HANDLER);
+  }
+
+  @Test
+  public void testAwaitAlwaysFalse() throws Throwable {
+    try {
+      await(TIMEOUT,
+          ALWAYS_FALSE,
+          retry,
+          TIMEOUT_FAILURE_HANDLER);
+      fail("should not have got here");
+    } catch (TimeoutException e) {
+      assertTrue(retry.getInvocationCount() > 4);
+    }
+  }
+
+  @Test
+  public void testAwaitLinearRetry() throws Throwable {
+    ProportionalRetryInterval linearRetry =
+        new ProportionalRetryInterval(INTERVAL * 2, TIMEOUT * 2);
+    try {
+      await(TIMEOUT,
+          ALWAYS_FALSE,
+          linearRetry,
+          TIMEOUT_FAILURE_HANDLER);
+      fail("should not have got here");
+    } catch (TimeoutException e) {
+      assertEquals(linearRetry.toString(),
+          2, linearRetry.getInvocationCount());
+    }
+  }
+
+  @Test
+  public void testAwaitFNFE() throws Throwable {
+    try {
+      await(TIMEOUT,
+          ALWAYS_FNFE,
+          retry,
+          TIMEOUT_FAILURE_HANDLER);
+      fail("should not have got here");
+    } catch (TimeoutException e) {
+      // inner clause is included
+      assertTrue(retry.getInvocationCount() > 0);
+      assertTrue(e.getCause() instanceof FileNotFoundException);
+      assertExceptionContains(MISSING, e);
+    }
+  }
+
+  @Test
+  public void testRetryInterval() throws Throwable {
+    ProportionalRetryInterval interval =
+        new ProportionalRetryInterval(200, 1000);
+    assertEquals(200, (int) interval.call());
+    assertEquals(400, (int) interval.call());
+    assertEquals(600, (int) interval.call());
+    assertEquals(800, (int) interval.call());
+    assertEquals(1000, (int) interval.call());
+    assertEquals(1000, (int) interval.call());
+    assertEquals(1000, (int) interval.call());
+  }
+
+  @Test
+  public void testInterceptSuccess() throws Throwable {
+    IOException ioe = intercept(IOException.class, ALWAYS_FNFE);
+    assertExceptionContains(MISSING, ioe);
+  }
+
+  @Test
+  public void testInterceptContains() throws Throwable {
+    intercept(IOException.class, MISSING, ALWAYS_FNFE);
+  }
+
+  @Test
+  public void testInterceptContainsWrongString() throws Throwable {
+    try {
+      FileNotFoundException e =
+          intercept(FileNotFoundException.class, "404", ALWAYS_FNFE);
+      assertNotNull(e);
+      throw e;
+    } catch (AssertionError expected) {
+      assertExceptionContains(MISSING, expected);
+    }
+  }
+
+  @Test
+  public void testInterceptVoidCallable() throws Throwable {
+    intercept(AssertionError.class,
+        NULL_RESULT,
+        new Callable<IOException>() {
+          @Override
+          public IOException call() throws Exception {
+            return intercept(IOException.class,
+                new Callable<Void>() {
+                  @Override
+                  public Void call() throws Exception {
+                    return null;
+                  }
+                });
+          }
+        });
+  }
+
+  @Test
+  public void testEventually() throws Throwable {
+    long result = eventually(TIMEOUT, EVAL_3L, retry);
+    assertEquals(3, result);
+    assertEquals(0, retry.getInvocationCount());
+  }
+
+  @Test
+  public void testEventuallyFailuresRetry() throws Throwable {
+    try {
+      eventually(TIMEOUT, EVAL_FNFE, retry);
+      fail("should not have got here");
+    } catch (IOException expected) {
+      // expected
+      assertMinRetryCount(1);
+    }
+  }
+
+  /*
+   * Java 8 Examples go below this line.
+   */
+
+  @Test
+  public void testInterceptFailure() throws Throwable {
+    try {
+      IOException ioe = intercept(IOException.class, () -> "hello");
+      assertNotNull(ioe);
+      throw ioe;
+    } catch (AssertionError expected) {
+      assertExceptionContains("hello", expected);
+    }
+  }
+
+  @Test
+  public void testInterceptInterceptLambda() throws Throwable {
+    // here we use intercept() to test itself.
+    intercept(AssertionError.class,
+        MISSING,
+        () -> intercept(FileNotFoundException.class, "404", ALWAYS_FNFE));
+  }
+
+  @Test
+  public void testInterceptInterceptVoidResultLambda() throws Throwable {
+    // see what happens when a null is returned; type inference -> Void
+    intercept(AssertionError.class,
+        NULL_RESULT,
+        () -> intercept(IOException.class, () -> null));
+  }
+
+  @Test
+  public void testInterceptInterceptStringResultLambda() throws Throwable {
+    // see what happens when a string is returned; it should be used
+    // in the message
+    intercept(AssertionError.class,
+        "hello, world",
+        () -> intercept(IOException.class,
+            () -> "hello, world"));
+  }
+
+  @Test
+  public void testAwaitNoTimeoutLambda() throws Throwable {
+    await(0,
+        () -> true,
+        retry,
+        (timeout, ex) -> ex != null ? ex : new Exception("timeout"));
+    assertRetryCount(0);
+  }
+
+  @Test
+  public void testAwaitLambdaRepetitions() throws Throwable {
+    count = 0;
+
+    // lambda expression which will succeed after exactly 4 probes
+    int reps = await(TIMEOUT,
+        () -> ++count == 4,
+        () -> 10,
+        (timeout, ex) -> ex != null ? ex : new Exception("timeout"));
+    assertEquals(4, reps);
+  }
+
+  @Test
+  public void testInterceptAwaitLambdaException() throws Throwable {
+    count = 0;
+    IOException ioe = intercept(IOException.class,
+        () -> await(
+            TIMEOUT,
+            () -> {
+              throw new IOException("inner " + ++count);
+            },
+            retry,
+            (timeout, ex) -> ex));
+    assertRetryCount(count - 1);
+    // verify that the exception returned was the last one raised
+    assertExceptionContains(Integer.toString(count), ioe);
+  }
+
+  @Test
+  public void testInterceptAwaitLambdaDiagnostics() throws Throwable {
+    intercept(IOException.class, "generated",
+        () -> await(5,
+            () -> false,
+            () -> -1,  // force checks -1 timeout probes
+            (timeout, ex) -> new IOException("generated")));
+  }
+
+  @Test
+  public void testInterceptAwaitFailFastLambda() throws Throwable {
+    intercept(FailFastException.class,
+        () -> await(TIMEOUT,
+            () -> {
+              throw new FailFastException("ffe");
+            },
+            retry,
+            (timeout, ex) -> ex));
+    assertRetryCount(0);
+  }
+
+  @Test
+  public void testEventuallyOnceLambda() throws Throwable {
+    String result = eventually(0, () -> "hello", retry);
+    assertEquals("hello", result);
+    assertEquals(0, retry.getInvocationCount());
+  }
+
+  @Test
+  public void testEventuallyLambda() throws Throwable {
+    long result = eventually(TIMEOUT, () -> 3, retry);
+    assertEquals(3, result);
+    assertRetryCount(0);
+  }
+
+  @Test
+  public void testInterceptEventuallyLambdaFailures() throws Throwable {
+    intercept(IOException.class,
+        "oops",
+        () -> eventually(TIMEOUT,
+            () -> {
+              throw new IOException("oops");
+            },
+            retry));
+    assertMinRetryCount(1);
+  }
+
+  @Test
+  public void testInterceptEventuallyambdaFailuresNegativeRetry()
+      throws Throwable {
+    intercept(FileNotFoundException.class,
+        () -> eventually(TIMEOUT, EVAL_FNFE, () -> -1));
+  }
+
+  @Test
+  public void testInterceptEventuallyLambdaFailFast() throws Throwable {
+    intercept(FailFastException.class, "oops",
+        () -> eventually(
+            TIMEOUT,
+            () -> {
+              throw new FailFastException("oops");
+            },
+            retry));
+    assertRetryCount(0);
+  }
+
+}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java

@@ -32,7 +32,6 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.DiskChecker.DiskErrorException;
 
 public class TestDiskChecker {
@@ -192,25 +191,4 @@ public class TestDiskChecker {
     System.out.println("checkDir success: " + success);
 
   }
-
-  @Test (timeout = 30000)
-  public void testCheckDirsIOException() throws Throwable {
-    Path path = new Path("target", TestDiskChecker.class.getSimpleName());
-    File localDir = new File(path.toUri().getRawPath());
-    localDir.mkdir();
-    File localFile = new File(localDir, "test");
-    localFile.createNewFile();
-    File spyLocalDir = spy(localDir);
-    doReturn(localFile.toPath()).when(spyLocalDir).toPath();
-    try {
-      DiskChecker.checkDirs(spyLocalDir);
-      fail("Expected exception for I/O error");
-    } catch (DiskErrorException e) {
-      GenericTestUtils.assertExceptionContains("I/O error", e);
-      assertTrue(e.getCause() instanceof IOException);
-    } finally {
-      localFile.delete();
-      localDir.delete();
-    }
-  }
 }

hadoop-common-project/hadoop-common/src/test/resources/testConf.xml

@@ -859,7 +859,11 @@
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*in the specified format. Format accepts filesize in( )*</expected-output>
+          <expected-output>^( |\t)*in the specified format. Format accepts permissions in( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*octal \(%a\) and symbolic \(%A\), filesize in( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>

hadoop-common-project/hadoop-kms/dev-support/findbugsExcludeFile.xml

@@ -38,4 +38,11 @@
     <Class name="org.apache.hadoop.crypto.key.kms.server.KMSWebApp"/>
     <Bug pattern="DM_EXIT"/>
   </Match>
+  <!--
+    KMS wants to log the exception before it's thrown to tomcat and disappear.
+  -->
+  <Match>
+    <Class name="org.apache.hadoop.crypto.key.kms.server.KMS"/>
+    <Bug pattern="REC"/>
+  </Match>
 </FindBugsFilter>

hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java

@@ -145,9 +145,13 @@ public class KMSAuthenticationFilter
         requestURL.append("?").append(queryString);
       }
 
-      KMSWebApp.getKMSAudit().unauthenticated(
-          request.getRemoteHost(), method, requestURL.toString(),
-          kmsResponse.msg);
+      if (!method.equals("OPTIONS")) {
+        // an HTTP OPTIONS request is made as part of the SPNEGO authentication
+        // sequence. We do not need to audit log it, since it doesn't belong
+        // to KMS context. KMS server doesn't handle OPTIONS either.
+        KMSWebApp.getKMSAudit().unauthenticated(request.getRemoteHost(), method,
+            requestURL.toString(), kmsResponse.msg);
+      }
     }
   }
 

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java

@@ -1261,8 +1261,9 @@ public class DFSInputStream extends FSInputStream
      */
     if (ex instanceof InvalidBlockTokenException ||
         ex instanceof InvalidToken) {
-      DFSClient.LOG.info("Access token was invalid when connecting to "
-          + targetAddr + " : " + ex);
+      DFSClient.LOG.debug(
+          "Access token was invalid when connecting to {}: {}",
+          targetAddr, ex);
       return true;
     }
     return false;

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java

@@ -2478,11 +2478,12 @@ public class DistributedFileSystem extends FileSystem {
    */
   @Override
   public Path getTrashRoot(Path path) {
-    if ((path == null) || path.isRoot() || !dfs.isHDFSEncryptionEnabled()) {
+    if ((path == null) || !dfs.isHDFSEncryptionEnabled()) {
       return super.getTrashRoot(path);
     }
 
-    String parentSrc = path.getParent().toUri().getPath();
+    String parentSrc = path.isRoot()?
+        path.toUri().getPath():path.getParent().toUri().getPath();
     try {
       EncryptionZone ez = dfs.getEZForPath(parentSrc);
       if ((ez != null)) {

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java

@@ -284,7 +284,7 @@ public class DfsClientConf {
     return classes;
   }
 
-  private DataChecksum.Type getChecksumType(Configuration conf) {
+  private static DataChecksum.Type getChecksumType(Configuration conf) {
     final String checksum = conf.get(
         DFS_CHECKSUM_TYPE_KEY,
         DFS_CHECKSUM_TYPE_DEFAULT);
@@ -299,7 +299,7 @@ public class DfsClientConf {
   }
 
   // Construct a checksum option from conf
-  private ChecksumOpt getChecksumOptFromConf(Configuration conf) {
+  public static ChecksumOpt getChecksumOptFromConf(Configuration conf) {
     DataChecksum.Type type = getChecksumType(conf);
     int bytesPerChecksum = conf.getInt(DFS_BYTES_PER_CHECKSUM_KEY,
         DFS_BYTES_PER_CHECKSUM_DEFAULT);

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java

@@ -91,6 +91,7 @@ import org.apache.hadoop.io.retry.RetryPolicies;
 import org.apache.hadoop.io.retry.RetryPolicy;
 import org.apache.hadoop.io.retry.RetryUtils;
 import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.StandbyException;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.SecurityUtil;
@@ -471,6 +472,13 @@ public class WebHdfsFileSystem extends FileSystem
       }
 
       IOException re = JsonUtilClient.toRemoteException(m);
+
+      //check if exception is due to communication with a Standby name node
+      if (re.getMessage() != null && re.getMessage().endsWith(
+          StandbyException.class.getSimpleName())) {
+        LOG.trace("Detected StandbyException", re);
+        throw new IOException(re);
+      }
       // extract UGI-related exceptions and unwrap InvalidToken
       // the NN mangles these exceptions but the DN does not and may need
       // to re-fetch a token if either report the token is expired

hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/TestTokenAspect.java

@@ -34,7 +34,6 @@ import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -130,9 +129,8 @@ public class TestTokenAspect {
     }
 
     @Override
-    public FileStatus[] listStatus(Path f) throws FileNotFoundException,
-        IOException {
-      return null;
+    public FileStatus[] listStatus(Path f) throws IOException {
+      return new FileStatus[0];
     }
 
     @Override

hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java

@@ -69,7 +69,7 @@ class OpenFileCtxCache {
     Iterator<Entry<FileHandle, OpenFileCtx>> it = openFileMap.entrySet()
         .iterator();
     if (LOG.isTraceEnabled()) {
-      LOG.trace("openFileMap size:" + openFileMap.size());
+      LOG.trace("openFileMap size:" + size());
     }
 
     Entry<FileHandle, OpenFileCtx> idlest = null;
@@ -117,10 +117,10 @@ class OpenFileCtxCache {
   boolean put(FileHandle h, OpenFileCtx context) {
     OpenFileCtx toEvict = null;
     synchronized (this) {
-      Preconditions.checkState(openFileMap.size() <= this.maxStreams,
-          "stream cache size " + openFileMap.size()
-              + "  is larger than maximum" + this.maxStreams);
-      if (openFileMap.size() == this.maxStreams) {
+      Preconditions.checkState(size() <= this.maxStreams,
+          "stream cache size " + size() + "  is larger than maximum" + this
+              .maxStreams);
+      if (size() == this.maxStreams) {
         Entry<FileHandle, OpenFileCtx> pairs = getEntryToEvict();
         if (pairs ==null) {
           return false;
@@ -149,7 +149,7 @@ class OpenFileCtxCache {
     Iterator<Entry<FileHandle, OpenFileCtx>> it = openFileMap.entrySet()
         .iterator();
     if (LOG.isTraceEnabled()) {
-      LOG.trace("openFileMap size:" + openFileMap.size());
+      LOG.trace("openFileMap size:" + size());
     }
 
     while (it.hasNext()) {
@@ -168,7 +168,7 @@ class OpenFileCtxCache {
             openFileMap.remove(handle);
             if (LOG.isDebugEnabled()) {
               LOG.debug("After remove stream " + handle.getFileId()
-                  + ", the stream number:" + openFileMap.size());
+                  + ", the stream number:" + size());
             }
             ctxToRemove.add(ctx2);
           }
@@ -201,7 +201,7 @@ class OpenFileCtxCache {
       Iterator<Entry<FileHandle, OpenFileCtx>> it = openFileMap.entrySet()
           .iterator();
       if (LOG.isTraceEnabled()) {
-        LOG.trace("openFileMap size:" + openFileMap.size());
+        LOG.trace("openFileMap size:" + size());
       }
 
       while (it.hasNext()) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java

@@ -220,6 +220,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
       "dfs.namenode.reconstruction.pending.timeout-sec";
   public static final int     DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_DEFAULT = -1;
 
+  public static final String  DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY =
+      "dfs.namenode.maintenance.replication.min";
+  public static final int     DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_DEFAULT
+      = 1;
+
   public static final String  DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY =
       HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY;
   public static final int     DFS_NAMENODE_REPLICATION_MAX_STREAMS_DEFAULT = 2;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java

@@ -124,48 +124,57 @@ public class DFSUtil {
   }
 
   /**
-   * Compartor for sorting DataNodeInfo[] based on decommissioned states.
-   * Decommissioned nodes are moved to the end of the array on sorting with
-   * this compartor.
+   * Comparator for sorting DataNodeInfo[] based on
+   * decommissioned and entering_maintenance states.
    */
-  public static final Comparator<DatanodeInfo> DECOM_COMPARATOR = 
-    new Comparator<DatanodeInfo>() {
-      @Override
-      public int compare(DatanodeInfo a, DatanodeInfo b) {
-        return a.isDecommissioned() == b.isDecommissioned() ? 0 : 
-          a.isDecommissioned() ? 1 : -1;
+  public static class ServiceComparator implements Comparator<DatanodeInfo> {
+    @Override
+    public int compare(DatanodeInfo a, DatanodeInfo b) {
+      // Decommissioned nodes will still be moved to the end of the list
+      if (a.isDecommissioned()) {
+        return b.isDecommissioned() ? 0 : 1;
+      } else if (b.isDecommissioned()) {
+        return -1;
       }
-    };
 
+      // ENTERING_MAINTENANCE nodes should be after live nodes.
+      if (a.isEnteringMaintenance()) {
+        return b.isEnteringMaintenance() ? 0 : 1;
+      } else if (b.isEnteringMaintenance()) {
+        return -1;
+      } else {
+        return 0;
+      }
+    }
+  }
 
   /**
-   * Comparator for sorting DataNodeInfo[] based on decommissioned/stale states.
-   * Decommissioned/stale nodes are moved to the end of the array on sorting
-   * with this comparator.
-   */ 
+   * Comparator for sorting DataNodeInfo[] based on
+   * stale, decommissioned and entering_maintenance states.
+   * Order: live -> stale -> entering_maintenance -> decommissioned
+   */
   @InterfaceAudience.Private 
-  public static class DecomStaleComparator implements Comparator<DatanodeInfo> {
+  public static class ServiceAndStaleComparator extends ServiceComparator {
     private final long staleInterval;
 
     /**
-     * Constructor of DecomStaleComparator
+     * Constructor of ServiceAndStaleComparator
      * 
      * @param interval
      *          The time interval for marking datanodes as stale is passed from
      *          outside, since the interval may be changed dynamically
      */
-    public DecomStaleComparator(long interval) {
+    public ServiceAndStaleComparator(long interval) {
       this.staleInterval = interval;
     }
 
     @Override
     public int compare(DatanodeInfo a, DatanodeInfo b) {
-      // Decommissioned nodes will still be moved to the end of the list
-      if (a.isDecommissioned()) {
-        return b.isDecommissioned() ? 0 : 1;
-      } else if (b.isDecommissioned()) {
-        return -1;
+      int ret = super.compare(a, b);
+      if (ret != 0) {
+        return ret;
       }
+
       // Stale nodes will be moved behind the normal nodes
       boolean aStale = a.isStale(staleInterval);
       boolean bStale = b.isStale(staleInterval);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java

@@ -989,20 +989,17 @@ public class Dispatcher {
   }
 
   private boolean shouldIgnore(DatanodeInfo dn) {
-    // ignore decommissioned nodes
-    final boolean decommissioned = dn.isDecommissioned();
-    // ignore decommissioning nodes
-    final boolean decommissioning = dn.isDecommissionInProgress();
+    // ignore out-of-service nodes
+    final boolean outOfService = !dn.isInService();
     // ignore nodes in exclude list
     final boolean excluded = Util.isExcluded(excludedNodes, dn);
     // ignore nodes not in the include list (if include list is not empty)
     final boolean notIncluded = !Util.isIncluded(includedNodes, dn);
 
-    if (decommissioned || decommissioning || excluded || notIncluded) {
+    if (outOfService || excluded || notIncluded) {
       if (LOG.isTraceEnabled()) {
         LOG.trace("Excluding datanode " + dn
-            + ": decommissioned=" + decommissioned
-            + ", decommissioning=" + decommissioning
+            + ": outOfService=" + outOfService
             + ", excluded=" + excluded
             + ", notIncluded=" + notIncluded);
       }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java

@@ -126,6 +126,29 @@ import org.slf4j.LoggerFactory;
 
 /**
  * Keeps information related to the blocks stored in the Hadoop cluster.
+ * For block state management, it tries to maintain the  safety
+ * property of "# of live replicas == # of expected redundancy" under
+ * any events such as decommission, namenode failover, datanode failure.
+ *
+ * The motivation of maintenance mode is to allow admins quickly repair nodes
+ * without paying the cost of decommission. Thus with maintenance mode,
+ * # of live replicas doesn't have to be equal to # of expected redundancy.
+ * If any of the replica is in maintenance mode, the safety property
+ * is extended as follows. These property still apply for the case of zero
+ * maintenance replicas, thus we can use these safe property for all scenarios.
+ * a. # of live replicas >= # of min replication for maintenance.
+ * b. # of live replicas <= # of expected redundancy.
+ * c. # of live replicas and maintenance replicas >= # of expected redundancy.
+ *
+ * For regular replication, # of min live replicas for maintenance is determined
+ * by DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY. This number has to <=
+ * DFS_NAMENODE_REPLICATION_MIN_KEY.
+ * For erasure encoding, # of min live replicas for maintenance is
+ * BlockInfoStriped#getRealDataBlockNum.
+ *
+ * Another safety property is to satisfy the block placement policy. While the
+ * policy is configurable, the replicas the policy is applied to are the live
+ * replicas + maintenance replicas.
  */
 @InterfaceAudience.Private
 public class BlockManager implements BlockStatsMXBean {
@@ -341,6 +364,11 @@ public class BlockManager implements BlockStatsMXBean {
 
   private final BlockIdManager blockIdManager;
 
+  /** Minimum live replicas needed for the datanode to be transitioned
+   * from ENTERING_MAINTENANCE to IN_MAINTENANCE.
+   */
+  private final short minReplicationToBeInMaintenance;
+
   public BlockManager(final Namesystem namesystem, boolean haEnabled,
       final Configuration conf) throws IOException {
     this.namesystem = namesystem;
@@ -373,13 +401,13 @@ public class BlockManager implements BlockStatsMXBean {
     this.maxCorruptFilesReturned = conf.getInt(
       DFSConfigKeys.DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED_KEY,
       DFSConfigKeys.DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED);
-    this.defaultReplication = conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, 
-                                          DFSConfigKeys.DFS_REPLICATION_DEFAULT);
+    this.defaultReplication = conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY,
+        DFSConfigKeys.DFS_REPLICATION_DEFAULT);
 
-    final int maxR = conf.getInt(DFSConfigKeys.DFS_REPLICATION_MAX_KEY, 
-                                 DFSConfigKeys.DFS_REPLICATION_MAX_DEFAULT);
+    final int maxR = conf.getInt(DFSConfigKeys.DFS_REPLICATION_MAX_KEY,
+        DFSConfigKeys.DFS_REPLICATION_MAX_DEFAULT);
     final int minR = conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY,
-                                 DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_DEFAULT);
+        DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_DEFAULT);
     if (minR <= 0)
       throw new IOException("Unexpected configuration parameters: "
           + DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY
@@ -407,7 +435,7 @@ public class BlockManager implements BlockStatsMXBean {
     this.blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf);
     this.blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf);
 
-    this.replicationRecheckInterval = 
+    this.replicationRecheckInterval =
       conf.getTimeDuration(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
           DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT,
           TimeUnit.SECONDS) * 1000L;
@@ -428,7 +456,7 @@ public class BlockManager implements BlockStatsMXBean {
     this.encryptDataTransfer =
         conf.getBoolean(DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY,
             DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_DEFAULT);
-    
+
     this.maxNumBlocksToLog =
         conf.getLong(DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_KEY,
             DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_DEFAULT);
@@ -438,6 +466,25 @@ public class BlockManager implements BlockStatsMXBean {
     this.getBlocksMinBlockSize = conf.getLongBytes(
         DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY,
         DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_DEFAULT);
+
+    final int minMaintenanceR = conf.getInt(
+        DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY,
+        DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_DEFAULT);
+
+    if (minMaintenanceR < 0) {
+      throw new IOException("Unexpected configuration parameters: "
+          + DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY
+          + " = " + minMaintenanceR + " < 0");
+    }
+    if (minMaintenanceR > minR) {
+      throw new IOException("Unexpected configuration parameters: "
+          + DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY
+          + " = " + minMaintenanceR + " > "
+          + DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY
+          + " = " + minR);
+    }
+    this.minReplicationToBeInMaintenance = (short)minMaintenanceR;
+
     this.blockReportLeaseManager = new BlockReportLeaseManager(conf);
 
     bmSafeMode = new BlockManagerSafeMode(this, namesystem, haEnabled, conf);
@@ -668,7 +715,7 @@ public class BlockManager implements BlockStatsMXBean {
     // Dump all datanodes
     getDatanodeManager().datanodeDump(out);
   }
-  
+
   /**
    * Dump the metadata for the given block in a human-readable
    * form.
@@ -697,12 +744,12 @@ public class BlockManager implements BlockStatsMXBean {
       out.print(fileName + ": ");
     }
     // l: == live:, d: == decommissioned c: == corrupt e: == excess
-    out.print(block + ((usableReplicas > 0)? "" : " MISSING") + 
+    out.print(block + ((usableReplicas > 0)? "" : " MISSING") +
               " (replicas:" +
               " l: " + numReplicas.liveReplicas() +
               " d: " + numReplicas.decommissionedAndDecommissioning() +
               " c: " + numReplicas.corruptReplicas() +
-              " e: " + numReplicas.excessReplicas() + ") "); 
+              " e: " + numReplicas.excessReplicas() + ") ");
 
     Collection<DatanodeDescriptor> corruptNodes = 
                                   corruptReplicas.getNodes(block);
@@ -750,6 +797,18 @@ public class BlockManager implements BlockStatsMXBean {
     }
   }
 
+  public short getMinReplicationToBeInMaintenance() {
+    return minReplicationToBeInMaintenance;
+  }
+
+  private short getMinMaintenanceStorageNum(BlockInfo block) {
+    if (block.isStriped()) {
+      return ((BlockInfoStriped) block).getRealDataBlockNum();
+    } else {
+      return minReplicationToBeInMaintenance;
+    }
+  }
+
   public boolean hasMinStorage(BlockInfo block) {
     return countNodes(block).liveReplicas() >= getMinStorageNum(block);
   }
@@ -942,7 +1001,7 @@ public class BlockManager implements BlockStatsMXBean {
     NumberReplicas replicas = countNodes(lastBlock);
     neededReconstruction.remove(lastBlock, replicas.liveReplicas(),
         replicas.readOnlyReplicas(),
-        replicas.decommissionedAndDecommissioning(), getRedundancy(lastBlock));
+        replicas.outOfServiceReplicas(), getExpectedRedundancyNum(lastBlock));
     pendingReconstruction.remove(lastBlock);
 
     // remove this block from the list of pending blocks to be deleted. 
@@ -1078,7 +1137,8 @@ public class BlockManager implements BlockStatsMXBean {
     } else {
       isCorrupt = numCorruptReplicas != 0 && numCorruptReplicas == numNodes;
     }
-    final int numMachines = isCorrupt ? numNodes: numNodes - numCorruptReplicas;
+    int numMachines = isCorrupt ? numNodes: numNodes - numCorruptReplicas;
+    numMachines -= numReplicas.maintenanceNotForReadReplicas();
     DatanodeStorageInfo[] machines = new DatanodeStorageInfo[numMachines];
     final byte[] blockIndices = blk.isStriped() ? new byte[numMachines] : null;
     int j = 0, i = 0;
@@ -1086,11 +1146,17 @@ public class BlockManager implements BlockStatsMXBean {
       final boolean noCorrupt = (numCorruptReplicas == 0);
       for(DatanodeStorageInfo storage : blocksMap.getStorages(blk)) {
         if (storage.getState() != State.FAILED) {
+          final DatanodeDescriptor d = storage.getDatanodeDescriptor();
+          // Don't pick IN_MAINTENANCE or dead ENTERING_MAINTENANCE states.
+          if (d.isInMaintenance()
+              || (d.isEnteringMaintenance() && !d.isAlive())) {
+            continue;
+          }
+
           if (noCorrupt) {
             machines[j++] = storage;
             i = setBlockIndices(blk, blockIndices, i, storage);
           } else {
-            final DatanodeDescriptor d = storage.getDatanodeDescriptor();
             final boolean replicaCorrupt = isReplicaCorrupt(blk, d);
             if (isCorrupt || !replicaCorrupt) {
               machines[j++] = storage;
@@ -1106,7 +1172,7 @@ public class BlockManager implements BlockStatsMXBean {
     }
 
     assert j == machines.length :
-      "isCorrupt: " + isCorrupt + 
+      "isCorrupt: " + isCorrupt +
       " numMachines: " + numMachines +
       " numNodes: " + numNodes +
       " numCorrupt: " + numCorruptNodes +
@@ -1347,6 +1413,8 @@ public class BlockManager implements BlockStatsMXBean {
       }
     }
     checkSafeMode();
+    LOG.info("Removed blocks associated with storage {} from DataNode {}",
+        storageInfo, node);
   }
 
   /**
@@ -1698,8 +1766,11 @@ public class BlockManager implements BlockStatsMXBean {
     return scheduledWork;
   }
 
+  // Check if the number of live + pending replicas satisfies
+  // the expected redundancy.
   boolean hasEnoughEffectiveReplicas(BlockInfo block,
-      NumberReplicas numReplicas, int pendingReplicaNum, int required) {
+      NumberReplicas numReplicas, int pendingReplicaNum) {
+    int required = getExpectedLiveRedundancyNum(block, numReplicas);
     int numEffectiveReplicas = numReplicas.liveReplicas() + pendingReplicaNum;
     return (numEffectiveReplicas >= required) &&
         (pendingReplicaNum > 0 || isPlacementPolicySatisfied(block));
@@ -1714,8 +1785,6 @@ public class BlockManager implements BlockStatsMXBean {
       return null;
     }
 
-    short requiredRedundancy = getExpectedRedundancyNum(block);
-
     // get a source data-node
     List<DatanodeDescriptor> containingNodes = new ArrayList<>();
     List<DatanodeStorageInfo> liveReplicaNodes = new ArrayList<>();
@@ -1724,6 +1793,8 @@ public class BlockManager implements BlockStatsMXBean {
     final DatanodeDescriptor[] srcNodes = chooseSourceDatanodes(block,
         containingNodes, liveReplicaNodes, numReplicas,
         liveBlockIndices, priority);
+    short requiredRedundancy = getExpectedLiveRedundancyNum(block,
+        numReplicas);
     if(srcNodes == null || srcNodes.length == 0) {
       // block can not be reconstructed from any node
       LOG.debug("Block " + block + " cannot be reconstructed " +
@@ -1736,8 +1807,7 @@ public class BlockManager implements BlockStatsMXBean {
     assert liveReplicaNodes.size() >= numReplicas.liveReplicas();
 
     int pendingNum = pendingReconstruction.getNumReplicas(block);
-    if (hasEnoughEffectiveReplicas(block, numReplicas, pendingNum,
-        requiredRedundancy)) {
+    if (hasEnoughEffectiveReplicas(block, numReplicas, pendingNum)) {
       neededReconstruction.remove(block, priority);
       blockLog.debug("BLOCK* Removing {} from neededReconstruction as" +
           " it has enough replicas", block);
@@ -1761,9 +1831,11 @@ public class BlockManager implements BlockStatsMXBean {
 
       // should reconstruct all the internal blocks before scheduling
       // replication task for decommissioning node(s).
-      if (additionalReplRequired - numReplicas.decommissioning() > 0) {
-        additionalReplRequired = additionalReplRequired
-            - numReplicas.decommissioning();
+      if (additionalReplRequired - numReplicas.decommissioning() -
+          numReplicas.liveEnteringMaintenanceReplicas() > 0) {
+        additionalReplRequired = additionalReplRequired -
+            numReplicas.decommissioning() -
+            numReplicas.liveEnteringMaintenanceReplicas();
       }
       byte[] indices = new byte[liveBlockIndices.size()];
       for (int i = 0 ; i < liveBlockIndices.size(); i++) {
@@ -1805,11 +1877,11 @@ public class BlockManager implements BlockStatsMXBean {
     }
 
     // do not schedule more if enough replicas is already pending
-    final short requiredRedundancy = getExpectedRedundancyNum(block);
     NumberReplicas numReplicas = countNodes(block);
+    final short requiredRedundancy =
+        getExpectedLiveRedundancyNum(block, numReplicas);
     final int pendingNum = pendingReconstruction.getNumReplicas(block);
-    if (hasEnoughEffectiveReplicas(block, numReplicas, pendingNum,
-        requiredRedundancy)) {
+    if (hasEnoughEffectiveReplicas(block, numReplicas, pendingNum)) {
       neededReconstruction.remove(block, priority);
       rw.resetTargets();
       blockLog.debug("BLOCK* Removing {} from neededReplications as" +
@@ -1878,7 +1950,7 @@ public class BlockManager implements BlockStatsMXBean {
    * @throws IOException
    *           if the number of targets < minimum replication.
    * @see BlockPlacementPolicy#chooseTarget(String, int, Node,
-   *      Set, long, List, BlockStoragePolicy)
+   *      Set, long, List, BlockStoragePolicy, EnumSet)
    */
   public DatanodeStorageInfo[] chooseTarget4NewBlock(final String src,
       final int numOfReplicas, final Node client,
@@ -1985,13 +2057,15 @@ public class BlockManager implements BlockStatsMXBean {
         continue;
       }
 
-      // never use already decommissioned nodes or unknown state replicas
-      if (state == null || state == StoredReplicaState.DECOMMISSIONED) {
+      // never use already decommissioned nodes, maintenance node not
+      // suitable for read or unknown state replicas.
+      if (state == null || state == StoredReplicaState.DECOMMISSIONED
+          || state == StoredReplicaState.MAINTENANCE_NOT_FOR_READ) {
         continue;
       }
 
       if (priority != LowRedundancyBlocks.QUEUE_HIGHEST_PRIORITY
-          && !node.isDecommissionInProgress() 
+          && (!node.isDecommissionInProgress() && !node.isEnteringMaintenance())
           && node.getNumberOfBlocksToBeReplicated() >= maxReplicationStreams) {
         continue; // already reached replication limit
       }
@@ -2043,10 +2117,10 @@ public class BlockManager implements BlockStatsMXBean {
             continue;
           }
           NumberReplicas num = countNodes(timedOutItems[i]);
-          if (isNeededReconstruction(bi, num.liveReplicas())) {
+          if (isNeededReconstruction(bi, num)) {
             neededReconstruction.add(bi, num.liveReplicas(),
-                num.readOnlyReplicas(), num.decommissionedAndDecommissioning(),
-                getRedundancy(bi));
+                num.readOnlyReplicas(), num.outOfServiceReplicas(),
+                getExpectedRedundancyNum(bi));
           }
         }
       } finally {
@@ -2191,7 +2265,7 @@ public class BlockManager implements BlockStatsMXBean {
   public boolean processReport(final DatanodeID nodeID,
       final DatanodeStorage storage,
       final BlockListAsLongs newReport,
-      BlockReportContext context, boolean lastStorageInRpc) throws IOException {
+      BlockReportContext context) throws IOException {
     namesystem.writeLock();
     final long startTime = Time.monotonicNow(); //after acquiring write lock
     final long endTime;
@@ -2245,32 +2319,6 @@ public class BlockManager implements BlockStatsMXBean {
       }
       
       storageInfo.receivedBlockReport();
-      if (context != null) {
-        storageInfo.setLastBlockReportId(context.getReportId());
-        if (lastStorageInRpc) {
-          int rpcsSeen = node.updateBlockReportContext(context);
-          if (rpcsSeen >= context.getTotalRpcs()) {
-            long leaseId = blockReportLeaseManager.removeLease(node);
-            BlockManagerFaultInjector.getInstance().
-                removeBlockReportLease(node, leaseId);
-            List<DatanodeStorageInfo> zombies = node.removeZombieStorages();
-            if (zombies.isEmpty()) {
-              LOG.debug("processReport 0x{}: no zombie storages found.",
-                  Long.toHexString(context.getReportId()));
-            } else {
-              for (DatanodeStorageInfo zombie : zombies) {
-                removeZombieReplicas(context, zombie);
-              }
-            }
-            node.clearBlockReportContext();
-          } else {
-            LOG.debug("processReport 0x{}: {} more RPCs remaining in this " +
-                    "report.", Long.toHexString(context.getReportId()),
-                (context.getTotalRpcs() - rpcsSeen)
-            );
-          }
-        }
-      }
     } finally {
       endTime = Time.monotonicNow();
       namesystem.writeUnlock();
@@ -2295,36 +2343,25 @@ public class BlockManager implements BlockStatsMXBean {
     return !node.hasStaleStorages();
   }
 
-  private void removeZombieReplicas(BlockReportContext context,
-      DatanodeStorageInfo zombie) {
-    LOG.warn("processReport 0x{}: removing zombie storage {}, which no " +
-            "longer exists on the DataNode.",
-        Long.toHexString(context.getReportId()), zombie.getStorageID());
-    assert(namesystem.hasWriteLock());
-    Iterator<BlockInfo> iter = zombie.getBlockIterator();
-    int prevBlocks = zombie.numBlocks();
-    while (iter.hasNext()) {
-      BlockInfo block = iter.next();
-      // We assume that a block can be on only one storage in a DataNode.
-      // That's why we pass in the DatanodeDescriptor rather than the
-      // DatanodeStorageInfo.
-      // TODO: remove this assumption in case we want to put a block on
-      // more than one storage on a datanode (and because it's a difficult
-      // assumption to really enforce)
-      // DatanodeStorageInfo must be removed using the iterator to avoid
-      // ConcurrentModificationException in the underlying storage
-      iter.remove();
-      removeStoredBlock(block, zombie.getDatanodeDescriptor());
-      Block b = getBlockOnStorage(block, zombie);
-      if (b != null) {
-        invalidateBlocks.remove(zombie.getDatanodeDescriptor(), b);
+  public void removeBRLeaseIfNeeded(final DatanodeID nodeID,
+      final BlockReportContext context) throws IOException {
+    namesystem.writeLock();
+    DatanodeDescriptor node;
+    try {
+      node = datanodeManager.getDatanode(nodeID);
+      if (context != null) {
+        if (context.getTotalRpcs() == context.getCurRpc() + 1) {
+          long leaseId = this.getBlockReportLeaseManager().removeLease(node);
+          BlockManagerFaultInjector.getInstance().
+              removeBlockReportLease(node, leaseId);
+        }
+        LOG.debug("Processing RPC with index {} out of total {} RPCs in "
+                + "processReport 0x{}", context.getCurRpc(),
+            context.getTotalRpcs(), Long.toHexString(context.getReportId()));
       }
+    } finally {
+      namesystem.writeUnlock();
     }
-    assert(zombie.numBlocks() == 0);
-    LOG.warn("processReport 0x{}: removed {} replicas from storage {}, " +
-            "which no longer exists on the DataNode.",
-        Long.toHexString(context.getReportId()), prevBlocks,
-        zombie.getStorageID());
   }
 
   /**
@@ -3049,10 +3086,9 @@ public class BlockManager implements BlockStatsMXBean {
 
     // handle low redundancy/extra redundancy
     short fileRedundancy = getExpectedRedundancyNum(storedBlock);
-    if (!isNeededReconstruction(storedBlock, numCurrentReplica)) {
+    if (!isNeededReconstruction(storedBlock, num, pendingNum)) {
       neededReconstruction.remove(storedBlock, numCurrentReplica,
-          num.readOnlyReplicas(),
-          num.decommissionedAndDecommissioning(), fileRedundancy);
+          num.readOnlyReplicas(), num.outOfServiceReplicas(), fileRedundancy);
     } else {
       updateNeededReconstructions(storedBlock, curReplicaDelta, 0);
     }
@@ -3075,6 +3111,10 @@ public class BlockManager implements BlockStatsMXBean {
     return storedBlock;
   }
 
+  // If there is any maintenance replica, we don't have to restore
+  // the condition of live + maintenance == expected. We allow
+  // live + maintenance >= expected. The extra redundancy will be removed
+  // when the maintenance node changes to live.
   private boolean shouldProcessExtraRedundancy(NumberReplicas num,
       int expectedNum) {
     final int numCurrent = num.liveReplicas();
@@ -3290,9 +3330,9 @@ public class BlockManager implements BlockStatsMXBean {
     NumberReplicas num = countNodes(block);
     final int numCurrentReplica = num.liveReplicas();
     // add to low redundancy queue if need to be
-    if (isNeededReconstruction(block, numCurrentReplica)) {
+    if (isNeededReconstruction(block, num)) {
       if (neededReconstruction.add(block, numCurrentReplica,
-          num.readOnlyReplicas(), num.decommissionedAndDecommissioning(),
+          num.readOnlyReplicas(), num.outOfServiceReplicas(),
           expectedRedundancy)) {
         return MisReplicationResult.UNDER_REPLICATED;
       }
@@ -3325,9 +3365,9 @@ public class BlockManager implements BlockStatsMXBean {
 
     // update neededReconstruction priority queues
     b.setReplication(newRepl);
+    NumberReplicas num = countNodes(b);
     updateNeededReconstructions(b, 0, newRepl - oldRepl);
-
-    if (oldRepl > newRepl) {
+    if (shouldProcessExtraRedundancy(num, newRepl)) {
       processExtraRedundancyBlock(b, newRepl, null, null);
     }
   }
@@ -3353,14 +3393,14 @@ public class BlockManager implements BlockStatsMXBean {
       }
       final DatanodeDescriptor cur = storage.getDatanodeDescriptor();
       if (storage.areBlockContentsStale()) {
-        LOG.trace("BLOCK* processOverReplicatedBlock: Postponing {}"
+        LOG.trace("BLOCK* processExtraRedundancyBlock: Postponing {}"
             + " since storage {} does not yet have up-to-date information.",
             block, storage);
         postponeBlock(block);
         return;
       }
       if (!isExcess(cur, block)) {
-        if (!cur.isDecommissionInProgress() && !cur.isDecommissioned()) {
+        if (cur.isInService()) {
           // exclude corrupt replicas
           if (corruptNodes == null || !corruptNodes.contains(cur)) {
             nonExcess.add(storage);
@@ -3801,7 +3841,7 @@ public class BlockManager implements BlockStatsMXBean {
     return countNodes(b, false);
   }
 
-  private NumberReplicas countNodes(BlockInfo b, boolean inStartupSafeMode) {
+  NumberReplicas countNodes(BlockInfo b, boolean inStartupSafeMode) {
     NumberReplicas numberReplicas = new NumberReplicas();
     Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(b);
     if (b.isStriped()) {
@@ -3832,6 +3872,12 @@ public class BlockManager implements BlockStatsMXBean {
         s = StoredReplicaState.DECOMMISSIONING;
       } else if (node.isDecommissioned()) {
         s = StoredReplicaState.DECOMMISSIONED;
+      } else if (node.isMaintenance()) {
+        if (node.isInMaintenance() || !node.isAlive()) {
+          s = StoredReplicaState.MAINTENANCE_NOT_FOR_READ;
+        } else {
+          s = StoredReplicaState.MAINTENANCE_FOR_READ;
+        }
       } else if (isExcess(node, b)) {
         s = StoredReplicaState.EXCESS;
       } else {
@@ -3903,11 +3949,11 @@ public class BlockManager implements BlockStatsMXBean {
   }
   
   /**
-   * On stopping decommission, check if the node has excess replicas.
+   * On putting the node in service, check if the node has excess replicas.
    * If there are any excess replicas, call processExtraRedundancyBlock().
    * Process extra redundancy blocks only when active NN is out of safe mode.
    */
-  void processExtraRedundancyBlocksOnReCommission(
+  void processExtraRedundancyBlocksOnInService(
       final DatanodeDescriptor srcNode) {
     if (!isPopulatingReplQueues()) {
       return;
@@ -3916,7 +3962,7 @@ public class BlockManager implements BlockStatsMXBean {
     int numExtraRedundancy = 0;
     while(it.hasNext()) {
       final BlockInfo block = it.next();
-      int expectedReplication = this.getRedundancy(block);
+      int expectedReplication = this.getExpectedRedundancyNum(block);
       NumberReplicas num = countNodes(block);
       if (shouldProcessExtraRedundancy(num, expectedReplication)) {
         // extra redundancy block
@@ -3926,14 +3972,15 @@ public class BlockManager implements BlockStatsMXBean {
       }
     }
     LOG.info("Invalidated " + numExtraRedundancy
-        + " extra redundancy blocks on " + srcNode + " during recommissioning");
+        + " extra redundancy blocks on " + srcNode + " after it is in service");
   }
 
   /**
-   * Returns whether a node can be safely decommissioned based on its 
-   * liveness. Dead nodes cannot always be safely decommissioned.
+   * Returns whether a node can be safely decommissioned or in maintenance
+   * based on its liveness. Dead nodes cannot always be safely decommissioned
+   * or in maintenance.
    */
-  boolean isNodeHealthyForDecommission(DatanodeDescriptor node) {
+  boolean isNodeHealthyForDecommissionOrMaintenance(DatanodeDescriptor node) {
     if (!node.checkBlockReportReceived()) {
       LOG.info("Node {} hasn't sent its first block report.", node);
       return false;
@@ -3947,17 +3994,18 @@ public class BlockManager implements BlockStatsMXBean {
     if (pendingReconstructionBlocksCount == 0 &&
         lowRedundancyBlocksCount == 0) {
       LOG.info("Node {} is dead and there are no low redundancy" +
-          " blocks or blocks pending reconstruction. Safe to decommission.",
-          node);
+          " blocks or blocks pending reconstruction. Safe to decommission or",
+          " put in maintenance.", node);
       return true;
     }
 
     LOG.warn("Node {} is dead " +
-        "while decommission is in progress. Cannot be safely " +
-        "decommissioned since there is risk of reduced " +
-        "data durability or data loss. Either restart the failed node or" +
-        " force decommissioning by removing, calling refreshNodes, " +
-        "then re-adding to the excludes files.", node);
+        "while in {}. Cannot be safely " +
+        "decommissioned or be in maintenance since there is risk of reduced " +
+        "data durability or data loss. Either restart the failed node or " +
+        "force decommissioning or maintenance by removing, calling " +
+        "refreshNodes, then re-adding to the excludes or host config files.",
+        node, node.getAdminState());
     return false;
   }
 
@@ -4025,17 +4073,16 @@ public class BlockManager implements BlockStatsMXBean {
       }
       NumberReplicas repl = countNodes(block);
       int pendingNum = pendingReconstruction.getNumReplicas(block);
-      int curExpectedReplicas = getRedundancy(block);
-      if (!hasEnoughEffectiveReplicas(block, repl, pendingNum,
-          curExpectedReplicas)) {
+      int curExpectedReplicas = getExpectedRedundancyNum(block);
+      if (!hasEnoughEffectiveReplicas(block, repl, pendingNum)) {
         neededReconstruction.update(block, repl.liveReplicas() + pendingNum,
-            repl.readOnlyReplicas(), repl.decommissionedAndDecommissioning(),
+            repl.readOnlyReplicas(), repl.outOfServiceReplicas(),
             curExpectedReplicas, curReplicasDelta, expectedReplicasDelta);
       } else {
         int oldReplicas = repl.liveReplicas() + pendingNum - curReplicasDelta;
         int oldExpectedReplicas = curExpectedReplicas-expectedReplicasDelta;
         neededReconstruction.remove(block, oldReplicas, repl.readOnlyReplicas(),
-            repl.decommissionedAndDecommissioning(), oldExpectedReplicas);
+            repl.outOfServiceReplicas(), oldExpectedReplicas);
       }
     } finally {
       namesystem.writeUnlock();
@@ -4053,24 +4100,15 @@ public class BlockManager implements BlockStatsMXBean {
       short expected = getExpectedRedundancyNum(block);
       final NumberReplicas n = countNodes(block);
       final int pending = pendingReconstruction.getNumReplicas(block);
-      if (!hasEnoughEffectiveReplicas(block, n, pending, expected)) {
+      if (!hasEnoughEffectiveReplicas(block, n, pending)) {
         neededReconstruction.add(block, n.liveReplicas() + pending,
-            n.readOnlyReplicas(),
-            n.decommissionedAndDecommissioning(), expected);
+            n.readOnlyReplicas(), n.outOfServiceReplicas(), expected);
       } else if (shouldProcessExtraRedundancy(n, expected)) {
         processExtraRedundancyBlock(block, expected, null, null);
       }
     }
   }
 
-  /** 
-   * @return 0 if the block is not found;
-   *         otherwise, return the replication factor of the block.
-   */
-  private int getRedundancy(BlockInfo block) {
-    return getExpectedRedundancyNum(block);
-  }
-
   /**
    * Get blocks to invalidate for <i>nodeId</i>
    * in {@link #invalidateBlocks}.
@@ -4123,6 +4161,8 @@ public class BlockManager implements BlockStatsMXBean {
         .getNodes(storedBlock);
     for (DatanodeStorageInfo storage : blocksMap.getStorages(storedBlock)) {
       final DatanodeDescriptor cur = storage.getDatanodeDescriptor();
+      // Nodes under maintenance should be counted as valid replicas from
+      // rack policy point of view.
       if (!cur.isDecommissionInProgress() && !cur.isDecommissioned()
           && ((corruptNodes == null) || !corruptNodes.contains(cur))) {
         liveNodes.add(cur);
@@ -4137,14 +4177,36 @@ public class BlockManager implements BlockStatsMXBean {
         .isPlacementPolicySatisfied();
   }
 
+  boolean isNeededReconstructionForMaintenance(BlockInfo storedBlock,
+      NumberReplicas numberReplicas) {
+    return storedBlock.isComplete() && (numberReplicas.liveReplicas() <
+        getMinMaintenanceStorageNum(storedBlock) ||
+        !isPlacementPolicySatisfied(storedBlock));
+  }
+
+  boolean isNeededReconstruction(BlockInfo storedBlock,
+      NumberReplicas numberReplicas) {
+    return isNeededReconstruction(storedBlock, numberReplicas, 0);
+  }
+
   /**
    * A block needs reconstruction if the number of redundancies is less than
    * expected or if it does not have enough racks.
    */
-  boolean isNeededReconstruction(BlockInfo storedBlock, int current) {
-    int expected = getExpectedRedundancyNum(storedBlock);
-    return storedBlock.isComplete()
-        && (current < expected || !isPlacementPolicySatisfied(storedBlock));
+  boolean isNeededReconstruction(BlockInfo storedBlock,
+      NumberReplicas numberReplicas, int pending) {
+    return storedBlock.isComplete() &&
+        !hasEnoughEffectiveReplicas(storedBlock, numberReplicas, pending);
+  }
+
+  // Exclude maintenance, but make sure it has minimal live replicas
+  // to satisfy the maintenance requirement.
+  public short getExpectedLiveRedundancyNum(BlockInfo block,
+      NumberReplicas numberReplicas) {
+    final short expectedRedundancy = getExpectedRedundancyNum(block);
+    return (short)Math.max(expectedRedundancy -
+        numberReplicas.maintenanceReplicas(),
+        getMinMaintenanceStorageNum(block));
   }
 
   public short getExpectedRedundancyNum(BlockInfo block) {