Merge branch 'trunk' into HDFS-7240

dev-support/docker/Dockerfile

@@ -18,21 +18,28 @@
 # Dockerfile for installing the necessary dependencies for building Hadoop.
 # See BUILDING.txt.
 
-
-FROM ubuntu:trusty
+FROM ubuntu:xenial
 
 WORKDIR /root
 
+#####
+# Disable suggests/recommends
+#####
+RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
+RUN echo APT::Install-Suggests "0"\; >>  /etc/apt/apt.conf.d/10disableextras
+
 ENV DEBIAN_FRONTEND noninteractive
 ENV DEBCONF_TERSE true
 
 ######
-# Install common dependencies from packages
+# Install common dependencies from packages. Versions here are either
+# sufficient or irrelevant.
 #
 # WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
 # Ubuntu Java.  See Java section below!
 ######
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
+RUN apt-get -q update && apt-get -q install -y \
+    apt-utils \
     build-essential \
     bzip2 \
     curl \
@@ -42,7 +49,6 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
     gcc \
     git \
     gnupg-agent \
-    make \
     libbz2-dev \
     libcurl4-openssl-dev \
     libfuse-dev \
@@ -51,106 +57,110 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
     libsnappy-dev \
     libssl-dev \
     libtool \
+    locales \
+    make \
     pinentry-curses \
     pkg-config \
-    protobuf-compiler \
-    protobuf-c-compiler \
     python \
     python2.7 \
-    python2.7-dev \
     python-pip \
+    python-pkg-resources \
+    python-setuptools \
+    python-wheel \
     rsync \
+    software-properties-common \
     snappy \
+    sudo \
     zlib1g-dev
 
 #######
-# Oracle Java
+# OpenJDK 8
 #######
+RUN apt-get -q install -y openjdk-8-jdk
 
-RUN echo "dot_style = mega" > "/root/.wgetrc"
-RUN echo "quiet = on" >> "/root/.wgetrc"
-
-RUN apt-get -q install --no-install-recommends -y software-properties-common
-RUN add-apt-repository -y ppa:webupd8team/java
-RUN apt-get -q update
-
-# Auto-accept the Oracle JDK license
-RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
-RUN apt-get -q install --no-install-recommends -y oracle-java8-installer
-
-####
-# Apps that require Java
-###
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y ant
+#######
+# OpenJDK 9
+# w/workaround for
+# https://bugs.launchpad.net/ubuntu/+source/openjdk-9/+bug/1593191
+#######
+RUN apt-get -o Dpkg::Options::="--force-overwrite" \
+    -q install -y \
+    openjdk-9-jdk-headless
 
-######
-# Install Apache Maven
-######
-RUN mkdir -p /opt/maven && \
-    curl -L -s -S \
-         https://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz \
-         -o /opt/maven.tar.gz && \
-    tar xzf /opt/maven.tar.gz --strip-components 1 -C /opt/maven
-ENV MAVEN_HOME /opt/maven
-ENV PATH "${PATH}:/opt/maven/bin"
+#######
+# Set default Java
+#######
+#
+# By default, OpenJDK sets the default Java to the highest version.
+# We want the opposite, soooooo....
+#
+RUN update-java-alternatives --set java-1.8.0-openjdk-amd64
+RUN update-alternatives --get-selections | grep -i jdk | \
+    while read line; do \
+      alternative=$(echo $line | awk '{print $1}'); \
+      path=$(echo $line | awk '{print $3}'); \
+      newpath=$(echo $path | sed -e 's/java-9/java-8/'); \
+      update-alternatives --set $alternative $newpath; \
+    done
 
 ######
-# Install cmake
+# Install cmake 3.1.0 (3.5.1 ships with Xenial)
 ######
 RUN mkdir -p /opt/cmake && \
     curl -L -s -S \
-         https://cmake.org/files/v3.1/cmake-3.1.0-Linux-x86_64.tar.gz \
-         -o /opt/cmake.tar.gz && \
+      https://cmake.org/files/v3.1/cmake-3.1.0-Linux-x86_64.tar.gz \
+      -o /opt/cmake.tar.gz && \
     tar xzf /opt/cmake.tar.gz --strip-components 1 -C /opt/cmake
 ENV CMAKE_HOME /opt/cmake
 ENV PATH "${PATH}:/opt/cmake/bin"
 
 ######
-# Install findbugs
+# Install Google Protobuf 2.5.0 (2.6.0 ships with Xenial)
 ######
-RUN mkdir -p /opt/findbugs && \
+RUN mkdir -p /opt/protobuf-src && \
     curl -L -s -S \
-         https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \
-         -o /opt/findbugs.tar.gz && \
-    tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs
-ENV FINDBUGS_HOME /opt/findbugs
-ENV PATH "${PATH}:/opt/findbugs/bin"
+      https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \
+      -o /opt/protobuf.tar.gz && \
+    tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src
+RUN cd /opt/protobuf-src && ./configure --prefix=/opt/protobuf && make install
+ENV PROTOBUF_HOME /opt/protobuf
+ENV PATH "${PATH}:/opt/protobuf/bin"
+
+######
+# Install Apache Maven 3.3.9 (3.3.9 ships with Xenial)
+######
+RUN apt-get -q update && apt-get -q install -y maven
+ENV MAVEN_HOME /usr
+
+######
+# Install findbugs 3.0.1 (3.0.1 ships with Xenial)
+######
+RUN apt-get -q update && apt-get -q install -y findbugs
+ENV FINDBUGS_HOME /usr
 
 ####
-# Install shellcheck
+# Install shellcheck (0.4.6, the latest as of 2017-09-26)
 ####
-RUN apt-get -q install -y cabal-install
-RUN mkdir /root/.cabal
-RUN echo "remote-repo: hackage.fpcomplete.com:http://hackage.fpcomplete.com/" >> /root/.cabal/config
-#RUN echo "remote-repo: hackage.haskell.org:http://hackage.haskell.org/" > /root/.cabal/config
-RUN echo "remote-repo-cache: /root/.cabal/packages" >> /root/.cabal/config
-RUN cabal update
-RUN cabal install shellcheck --global
+RUN add-apt-repository -y ppa:jonathonf/ghc-8.0.2
+RUN apt-get -q update && apt-get -q install -y shellcheck
 
 ####
-# Install bats
+# Install bats (0.4.0, the latest as of 2017-09-26, ships with Xenial)
 ####
-RUN add-apt-repository -y ppa:duggan/bats
-RUN apt-get -q update
-RUN apt-get -q install --no-install-recommends -y bats
+RUN apt-get -q update && apt-get -q install -y bats
 
 ####
-# Install pylint
+# Install pylint (always want latest)
 ####
-RUN pip install pylint
+RUN pip2 install pylint
 
 ####
 # Install dateutil.parser
 ####
-RUN pip install python-dateutil
+RUN pip2 install python-dateutil
 
 ###
-# Avoid out of memory errors in builds
-###
-ENV MAVEN_OPTS -Xms256m -Xmx512m
-
-###
-# Install node js tools for web UI frameowkr
+# Install node.js for web UI framework (4.2.6 ships with Xenial)
 ###
 RUN apt-get -y install nodejs && \
     ln -s /usr/bin/nodejs /usr/bin/node && \
@@ -159,6 +169,12 @@ RUN apt-get -y install nodejs && \
     npm install -g bower && \
     npm install -g ember-cli
 
+###
+# Avoid out of memory errors in builds
+###
+ENV MAVEN_OPTS -Xms256m -Xmx1g
+
+
 ###
 # Everything past this point is either not needed for testing or breaks Yetus.
 # So tell Yetus not to read the rest of the file:
@@ -166,12 +182,17 @@ RUN apt-get -y install nodejs && \
 ###
 
 ####
-# Install Forrest (for Apache Hadoop website)
+# Install svn, Ant, & Forrest (for Apache Hadoop website)
 ###
-RUN mkdir -p /usr/local/apache-forrest ; \
-    curl -s -S -O http://archive.apache.org/dist/forrest/0.8/apache-forrest-0.8.tar.gz ; \
-    tar xzf *forrest* --strip-components 1 -C /usr/local/apache-forrest ; \
-    echo 'forrest.home=/usr/local/apache-forrest' > build.properties
+RUN apt-get -q update && apt-get -q install -y ant subversion
+
+RUN mkdir -p /opt/apache-forrest && \
+    curl -L -s -S \
+      https://archive.apache.org/dist/forrest/0.8/apache-forrest-0.8.tar.gz \
+      -o /opt/forrest.tar.gz && \
+    tar xzf /opt/forrest.tar.gz --strip-components 1 -C /opt/apache-forrest
+RUN echo 'forrest.home=/opt/apache-forrest' > build.properties
+ENV FORREST_HOME=/opt/apache-forrest
 
 # Add a welcome message and environment checks.
 ADD hadoop_env_checks.sh /root/hadoop_env_checks.sh

dev-support/docker/hadoop_env_checks.sh

@@ -16,6 +16,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# SHELLDOC-IGNORE
+
 # -------------------------------------------------------
 function showWelcome {
 cat <<Welcome-message
@@ -80,14 +82,10 @@ End-of-message
 
 # -------------------------------------------------------
 
-# Configurable low water mark in GiB
-MINIMAL_MEMORY_GiB=2
-
 function warnIfLowMemory {
-    MINIMAL_MEMORY=$((MINIMAL_MEMORY_GiB*1024*1024)) # Convert to KiB
-    INSTALLED_MEMORY=$(fgrep MemTotal /proc/meminfo | awk '{print $2}')
-    if [ $((INSTALLED_MEMORY)) -le $((MINIMAL_MEMORY)) ];
-    then
+    MINIMAL_MEMORY=2046755
+    INSTALLED_MEMORY=$(grep -F MemTotal /proc/meminfo | awk '{print $2}')
+    if [[ $((INSTALLED_MEMORY)) -lt $((MINIMAL_MEMORY)) ]]; then
         cat <<End-of-message
 
  _                    ___  ___
@@ -103,7 +101,8 @@ Your system is running on very little memory.
 This means it may work but it wil most likely be slower than needed.
 
 If you are running this via boot2docker you can simply increase
-the available memory to atleast ${MINIMAL_MEMORY_GiB} GiB (you have $((INSTALLED_MEMORY/(1024*1024))) GiB )
+the available memory to at least ${MINIMAL_MEMORY}KiB
+(you have ${INSTALLED_MEMORY}KiB )
 
 End-of-message
     fi

hadoop-client-modules/hadoop-client/pom.xml

@@ -177,6 +177,66 @@
       </exclusions>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-client</artifactId>
+      <scope>compile</scope>
+      <exclusions>
+        <!--Excluding hadoop-yarn-api & hadoop-annotations as they are already
+        included as direct dependencies. Guava,commons-cli and log4j are
+        transitive dependencies -->
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-annotations</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-cli</groupId>
+          <artifactId>commons-cli</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>log4j</groupId>
+          <artifactId>log4j</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-server</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-json</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-servlet</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>io.netty</groupId>
+          <artifactId>netty</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.inject.extensions</groupId>
+          <artifactId>guice-servlet</artifactId>
+        </exclusion>
+
+      </exclusions>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-mapreduce-client-core</artifactId>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java

@@ -290,9 +290,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
 
   /**
    * Stores the mapping of key to the resource which modifies or loads 
-   * the key most recently
+   * the key most recently. Created lazily to avoid wasting memory.
    */
-  private Map<String, String[]> updatingResource;
+  private volatile Map<String, String[]> updatingResource;
 
   /**
    * Specify exact input factory to avoid time finding correct one.
@@ -749,7 +749,6 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
    */
   public Configuration(boolean loadDefaults) {
     this.loadDefaults = loadDefaults;
-    updatingResource = new ConcurrentHashMap<String, String[]>();
 
     // Register all classes holding property tags with
     REGISTERED_TAG_CLASS.put("core", CorePropertyTag.class);
@@ -769,25 +768,27 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
    */
   @SuppressWarnings("unchecked")
   public Configuration(Configuration other) {
-   this.resources = (ArrayList<Resource>) other.resources.clone();
-   synchronized(other) {
-     if (other.properties != null) {
-       this.properties = (Properties)other.properties.clone();
-     }
-
-     if (other.overlay!=null) {
-       this.overlay = (Properties)other.overlay.clone();
-     }
-
-     this.updatingResource = new ConcurrentHashMap<String, String[]>(
-         other.updatingResource);
-     this.finalParameters = Collections.newSetFromMap(
-         new ConcurrentHashMap<String, Boolean>());
-     this.finalParameters.addAll(other.finalParameters);
-     this.REGISTERED_TAG_CLASS.putAll(other.REGISTERED_TAG_CLASS);
-     this.propertyTagsMap.putAll(other.propertyTagsMap);
-   }
-   
+    this.resources = (ArrayList<Resource>) other.resources.clone();
+    synchronized(other) {
+      if (other.properties != null) {
+        this.properties = (Properties)other.properties.clone();
+      }
+
+      if (other.overlay!=null) {
+        this.overlay = (Properties)other.overlay.clone();
+      }
+
+      if (other.updatingResource != null) {
+        this.updatingResource = new ConcurrentHashMap<String, String[]>(
+           other.updatingResource);
+      }
+      this.finalParameters = Collections.newSetFromMap(
+          new ConcurrentHashMap<String, Boolean>());
+      this.finalParameters.addAll(other.finalParameters);
+      this.REGISTERED_TAG_CLASS.putAll(other.REGISTERED_TAG_CLASS);
+      this.propertyTagsMap.putAll(other.propertyTagsMap);
+    }
+
     synchronized(Configuration.class) {
       REGISTRY.put(this, null);
     }
@@ -1278,14 +1279,14 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     String newSource = (source == null ? "programmatically" : source);
 
     if (!isDeprecated(name)) {
-      updatingResource.put(name, new String[] {newSource});
+      putIntoUpdatingResource(name, new String[] {newSource});
       String[] altNames = getAlternativeNames(name);
       if(altNames != null) {
         for(String n: altNames) {
           if(!n.equals(name)) {
             getOverlay().setProperty(n, value);
             getProps().setProperty(n, value);
-            updatingResource.put(n, new String[] {newSource});
+            putIntoUpdatingResource(n, new String[] {newSource});
           }
         }
       }
@@ -1296,7 +1297,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
       for(String n : names) {
         getOverlay().setProperty(n, value);
         getProps().setProperty(n, value);
-        updatingResource.put(n, new String[] {altSource});
+        putIntoUpdatingResource(n, new String[] {altSource});
       }
     }
   }
@@ -2635,17 +2636,19 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
   protected synchronized Properties getProps() {
     if (properties == null) {
       properties = new Properties();
-      Map<String, String[]> backup =
-          new ConcurrentHashMap<String, String[]>(updatingResource);
+      Map<String, String[]> backup = updatingResource != null ?
+          new ConcurrentHashMap<String, String[]>(updatingResource) : null;
       loadResources(properties, resources, quietmode);
 
       if (overlay != null) {
         properties.putAll(overlay);
-        for (Map.Entry<Object,Object> item: overlay.entrySet()) {
-          String key = (String)item.getKey();
-          String[] source = backup.get(key);
-          if(source != null) {
-            updatingResource.put(key, source);
+        if (backup != null) {
+          for (Map.Entry<Object, Object> item : overlay.entrySet()) {
+            String key = (String) item.getKey();
+            String[] source = backup.get(key);
+            if (source != null) {
+              updatingResource.put(key, source);
+            }
           }
         }
       }
@@ -2701,11 +2704,14 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
    * @return mapping of configuration properties with prefix stripped
    */
   public Map<String, String> getPropsWithPrefix(String confPrefix) {
+    Properties props = getProps();
+    Enumeration e = props.propertyNames();
     Map<String, String> configMap = new HashMap<>();
-    for (Map.Entry<String, String> entry : this) {
-      String name = entry.getKey();
+    String name = null;
+    while (e.hasMoreElements()) {
+      name = (String) e.nextElement();
       if (name.startsWith(confPrefix)) {
-        String value = this.get(name);
+        String value = props.getProperty(name);
         name = name.substring(confPrefix.length());
         configMap.put(name, value);
       }
@@ -3058,8 +3064,8 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
       }
       if (!finalParameters.contains(attr)) {
         properties.setProperty(attr, value);
-        if(source != null) {
-          updatingResource.put(attr, source);
+        if (source != null) {
+          putIntoUpdatingResource(attr, source);
         }
       } else {
         // This is a final parameter so check for overrides.
@@ -3364,9 +3370,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
           redactor.redact(name, config.get(name)));
       jsonGen.writeBooleanField("isFinal",
           config.finalParameters.contains(name));
-      String[] resources = config.updatingResource.get(name);
+      String[] resources = config.updatingResource != null ?
+          config.updatingResource.get(name) : null;
       String resource = UNKNOWN_RESOURCE;
-      if(resources != null && resources.length > 0) {
+      if (resources != null && resources.length > 0) {
         resource = resources[0];
       }
       jsonGen.writeStringField("resource", resource);
@@ -3446,8 +3453,8 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
       String value = org.apache.hadoop.io.Text.readString(in);
       set(key, value); 
       String sources[] = WritableUtils.readCompressedStringArray(in);
-      if(sources != null) {
-        updatingResource.put(key, sources);
+      if (sources != null) {
+        putIntoUpdatingResource(key, sources);
       }
     }
   }
@@ -3460,8 +3467,8 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     for(Map.Entry<Object, Object> item: props.entrySet()) {
       org.apache.hadoop.io.Text.writeString(out, (String) item.getKey());
       org.apache.hadoop.io.Text.writeString(out, (String) item.getValue());
-      WritableUtils.writeCompressedStringArray(out, 
-          updatingResource.get(item.getKey()));
+      WritableUtils.writeCompressedStringArray(out, updatingResource != null ?
+          updatingResource.get(item.getKey()) : null);
     }
   }
   
@@ -3561,4 +3568,17 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     }
     return tag;
   }
+
+  private void putIntoUpdatingResource(String key, String[] value) {
+    Map<String, String[]> localUR = updatingResource;
+    if (localUR == null) {
+      synchronized (this) {
+        localUR = updatingResource;
+        if (localUR == null) {
+          updatingResource = localUR = new ConcurrentHashMap<>(8);
+        }
+      }
+    }
+    localUR.put(key, value);
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java

@@ -121,10 +121,6 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
 
   private static final String CONFIG_PREFIX = "hadoop.security.kms.client.";
 
-  /* It's possible to specify a timeout, in seconds, in the config file */
-  public static final String TIMEOUT_ATTR = CONFIG_PREFIX + "timeout";
-  public static final int DEFAULT_TIMEOUT = 60;
-
   /* Number of times to retry authentication in the event of auth failure
    * (normally happens due to stale authToken) 
    */
@@ -361,7 +357,9 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
         throw new IOException(ex);
       }
     }
-    int timeout = conf.getInt(TIMEOUT_ATTR, DEFAULT_TIMEOUT);
+    int timeout = conf.getInt(
+            CommonConfigurationKeysPublic.KMS_CLIENT_TIMEOUT_SECONDS,
+            CommonConfigurationKeysPublic.KMS_CLIENT_TIMEOUT_DEFAULT);
     authRetry = conf.getInt(AUTH_RETRY, DEFAULT_AUTH_RETRY);
     configurator = new TimeoutConnConfigurator(timeout, sslFactory);
     encKeyVersionQueue =

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java

@@ -721,6 +721,15 @@ public class CommonConfigurationKeysPublic {
   /** Default value for KMS_CLIENT_ENC_KEY_CACHE_EXPIRY (12 hrs)*/
   public static final int KMS_CLIENT_ENC_KEY_CACHE_EXPIRY_DEFAULT = 43200000;
 
+  /**
+   * @see
+   * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+   * core-default.xml</a>
+   */
+  public static final String KMS_CLIENT_TIMEOUT_SECONDS =
+      "hadoop.security.kms.client.timeout";
+  public static final int KMS_CLIENT_TIMEOUT_DEFAULT = 60;
+
   /**
    * @see
    * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ConfigUtil.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.fs.viewfs;
 
 import java.net.URI;
+import java.util.Arrays;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.StringUtils;
@@ -68,7 +69,72 @@ public class ConfigUtil {
     addLink( conf, Constants.CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE, 
         src, target);   
   }
-  
+
+  /**
+   * Add a LinkMergeSlash to the config for the specified mount table.
+   * @param conf
+   * @param mountTableName
+   * @param target
+   */
+  public static void addLinkMergeSlash(Configuration conf,
+      final String mountTableName, final URI target) {
+    conf.set(getConfigViewFsPrefix(mountTableName) + "." +
+        Constants.CONFIG_VIEWFS_LINK_MERGE_SLASH, target.toString());
+  }
+
+  /**
+   * Add a LinkMergeSlash to the config for the default mount table.
+   * @param conf
+   * @param target
+   */
+  public static void addLinkMergeSlash(Configuration conf, final URI target) {
+    addLinkMergeSlash(conf, Constants.CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE,
+        target);
+  }
+
+  /**
+   * Add a LinkFallback to the config for the specified mount table.
+   * @param conf
+   * @param mountTableName
+   * @param target
+   */
+  public static void addLinkFallback(Configuration conf,
+      final String mountTableName, final URI target) {
+    conf.set(getConfigViewFsPrefix(mountTableName) + "." +
+        Constants.CONFIG_VIEWFS_LINK_FALLBACK, target.toString());
+  }
+
+  /**
+   * Add a LinkFallback to the config for the default mount table.
+   * @param conf
+   * @param target
+   */
+  public static void addLinkFallback(Configuration conf, final URI target) {
+    addLinkFallback(conf, Constants.CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE,
+        target);
+  }
+
+  /**
+   * Add a LinkMerge to the config for the specified mount table.
+   * @param conf
+   * @param mountTableName
+   * @param targets
+   */
+  public static void addLinkMerge(Configuration conf,
+      final String mountTableName, final URI[] targets) {
+    conf.set(getConfigViewFsPrefix(mountTableName) + "." +
+        Constants.CONFIG_VIEWFS_LINK_MERGE, Arrays.toString(targets));
+  }
+
+  /**
+   * Add a LinkMerge to the config for the default mount table.
+   * @param conf
+   * @param targets
+   */
+  public static void addLinkMerge(Configuration conf, final URI[] targets) {
+    addLinkMerge(conf, Constants.CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE, targets);
+  }
+
   /**
    *
    * @param conf

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java

@@ -51,12 +51,17 @@ public interface Constants {
   /**
    * Config variable for specifying a simple link
    */
-  public static final String CONFIG_VIEWFS_LINK = "link";
-  
+  String CONFIG_VIEWFS_LINK = "link";
+
+  /**
+   * Config variable for specifying a fallback for link mount points.
+   */
+  String CONFIG_VIEWFS_LINK_FALLBACK = "linkFallback";
+
   /**
    * Config variable for specifying a merge link
    */
-  public static final String CONFIG_VIEWFS_LINK_MERGE = "linkMerge";
+  String CONFIG_VIEWFS_LINK_MERGE = "linkMerge";
 
   /**
    * Config variable for specifying an nfly link. Nfly writes to multiple
@@ -68,10 +73,9 @@ public interface Constants {
    * Config variable for specifying a merge of the root of the mount-table
    *  with the root of another file system. 
    */
-  public static final String CONFIG_VIEWFS_LINK_MERGE_SLASH = "linkMergeSlash";
+  String CONFIG_VIEWFS_LINK_MERGE_SLASH = "linkMergeSlash";
 
-  static public final FsPermission PERMISSION_555 =
-      new FsPermission((short) 0555);
+  FsPermission PERMISSION_555 = new FsPermission((short) 0555);
 
   String CONFIG_VIEWFS_RENAME_STRATEGY = "fs.viewfs.rename.strategy";
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java

@@ -17,12 +17,15 @@
  */
 package org.apache.hadoop.fs.viewfs;
 
+import com.google.common.base.Preconditions;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -61,8 +64,12 @@ abstract class InodeTree<T> {
   }
 
   static final Path SlashPath = new Path("/");
-  private final INodeDir<T> root;     // the root of the mount table
-  private final String homedirPrefix; // the homedir for this mount table
+  // the root of the mount table
+  private final INode<T> root;
+  // the fallback filesystem
+  private final INodeLink<T> rootFallbackLink;
+  // the homedir for this mount table
+  private final String homedirPrefix;
   private List<MountPoint<T>> mountPoints = new ArrayList<MountPoint<T>>();
 
   static class MountPoint<T> {
@@ -85,7 +92,7 @@ abstract class InodeTree<T> {
   }
 
   /**
-   * Internal class for inode tree
+   * Internal class for INode tree.
    * @param <T>
    */
   abstract static class INode<T> {
@@ -94,21 +101,58 @@ abstract class InodeTree<T> {
     public INode(String pathToNode, UserGroupInformation aUgi) {
       fullPath = pathToNode;
     }
+
+    // INode forming the internal mount table directory tree
+    // for ViewFileSystem. This internal directory tree is
+    // constructed based on the mount table config entries
+    // and is read only.
+    abstract boolean isInternalDir();
+
+    // INode linking to another filesystem. Represented
+    // via mount table link config entries.
+    boolean isLink() {
+      return !isInternalDir();
+    }
   }
 
   /**
-   * Internal class to represent an internal dir of the mount table
+   * Internal class to represent an internal dir of the mount table.
    * @param <T>
    */
   static class INodeDir<T> extends INode<T> {
-    final Map<String,INode<T>> children = new HashMap<String,INode<T>>();
-    T InodeDirFs =  null; // file system of this internal directory of mountT
-    boolean isRoot = false;
+    private final Map<String, INode<T>> children = new HashMap<>();
+    private T internalDirFs =  null; //filesystem of this internal directory
+    private boolean isRoot = false;
 
     INodeDir(final String pathToNode, final UserGroupInformation aUgi) {
       super(pathToNode, aUgi);
     }
 
+    @Override
+    boolean isInternalDir() {
+      return true;
+    }
+
+    T getInternalDirFs() {
+      return internalDirFs;
+    }
+
+    void setInternalDirFs(T internalDirFs) {
+      this.internalDirFs = internalDirFs;
+    }
+
+    void setRoot(boolean root) {
+      isRoot = root;
+    }
+
+    boolean isRoot() {
+      return isRoot;
+    }
+
+    Map<String, INode<T>> getChildren() {
+      return Collections.unmodifiableMap(children);
+    }
+
     INode<T> resolveInternal(final String pathComponent) {
       return children.get(pathComponent);
     }
@@ -119,7 +163,7 @@ abstract class InodeTree<T> {
         throw new FileAlreadyExistsException();
       }
       final INodeDir<T> newDir = new INodeDir<T>(fullPath +
-          (isRoot ? "" : "/") + pathComponent, aUgi);
+          (isRoot() ? "" : "/") + pathComponent, aUgi);
       children.put(pathComponent, newDir);
       return newDir;
     }
@@ -133,10 +177,43 @@ abstract class InodeTree<T> {
     }
   }
 
+  /**
+   * Mount table link type.
+   */
   enum LinkType {
+    /**
+     * Link entry pointing to a single filesystem uri.
+     * Config prefix: fs.viewfs.mounttable.<mnt_tbl_name>.link.<link_name>
+     * Refer: {@link Constants#CONFIG_VIEWFS_LINK}
+     */
     SINGLE,
+    /**
+     * Fallback filesystem for the paths not mounted by
+     * any single link entries.
+     * Config prefix: fs.viewfs.mounttable.<mnt_tbl_name>.linkFallback
+     * Refer: {@link Constants#CONFIG_VIEWFS_LINK_FALLBACK}
+     */
+    SINGLE_FALLBACK,
+    /**
+     * Link entry pointing to an union of two or more filesystem uris.
+     * Config prefix: fs.viewfs.mounttable.<mnt_tbl_name>.linkMerge.<link_name>
+     * Refer: {@link Constants#CONFIG_VIEWFS_LINK_MERGE}
+     */
     MERGE,
-    NFLY
+    /**
+     * Link entry for merging mount table's root with the
+     * root of another filesystem.
+     * Config prefix: fs.viewfs.mounttable.<mnt_tbl_name>.linkMergeSlash
+     * Refer: {@link Constants#CONFIG_VIEWFS_LINK_MERGE_SLASH}
+     */
+    MERGE_SLASH,
+    /**
+     * Link entry to write to multiple filesystems and read
+     * from the closest filesystem.
+     * Config prefix: fs.viewfs.mounttable.<mnt_tbl_name>.linkNfly
+     * Refer: {@link Constants#CONFIG_VIEWFS_LINK_NFLY}
+     */
+    NFLY;
   }
 
   /**
@@ -188,6 +265,15 @@ abstract class InodeTree<T> {
       }
       return new Path(result.toString());
     }
+
+    @Override
+    boolean isInternalDir() {
+      return false;
+    }
+
+    public T getTargetFileSystem() {
+      return targetFileSystem;
+    }
   }
 
   private void createLink(final String src, final String target,
@@ -203,7 +289,10 @@ abstract class InodeTree<T> {
     }
 
     final String[] srcPaths = breakIntoPathComponents(src);
-    INodeDir<T> curInode = root;
+    // Make sure root is of INodeDir type before
+    // adding any regular links to it.
+    Preconditions.checkState(root.isInternalDir());
+    INodeDir<T> curInode = getRootDir();
     int i;
     // Ignore first initial slash, process all except last component
     for (i = 1; i < srcPaths.length - 1; i++) {
@@ -211,15 +300,15 @@ abstract class InodeTree<T> {
       INode<T> nextInode = curInode.resolveInternal(iPath);
       if (nextInode == null) {
         INodeDir<T> newDir = curInode.addDir(iPath, aUgi);
-        newDir.InodeDirFs = getTargetFileSystem(newDir);
+        newDir.setInternalDirFs(getTargetFileSystem(newDir));
         nextInode = newDir;
       }
-      if (nextInode instanceof INodeLink) {
+      if (nextInode.isLink()) {
         // Error - expected a dir but got a link
         throw new FileAlreadyExistsException("Path " + nextInode.fullPath +
             " already exists as link");
       } else {
-        assert (nextInode instanceof INodeDir);
+        assert(nextInode.isInternalDir());
         curInode = (INodeDir<T>) nextInode;
       }
     }
@@ -245,6 +334,11 @@ abstract class InodeTree<T> {
       newLink = new INodeLink<T>(fullPath, aUgi,
           getTargetFileSystem(new URI(target)), new URI(target));
       break;
+    case SINGLE_FALLBACK:
+    case MERGE_SLASH:
+      // Link fallback and link merge slash configuration
+      // are handled specially at InodeTree.
+      throw new IllegalArgumentException("Unexpected linkType: " + linkType);
     case MERGE:
     case NFLY:
       final URI[] targetUris = StringUtils.stringToURI(
@@ -273,6 +367,77 @@ abstract class InodeTree<T> {
   protected abstract T getTargetFileSystem(String settings, URI[] mergeFsURIs)
       throws UnsupportedFileSystemException, URISyntaxException, IOException;
 
+  private INodeDir<T> getRootDir() {
+    Preconditions.checkState(root.isInternalDir());
+    return (INodeDir<T>)root;
+  }
+
+  private INodeLink<T> getRootLink() {
+    Preconditions.checkState(root.isLink());
+    return (INodeLink<T>)root;
+  }
+
+  private boolean hasFallbackLink() {
+    return rootFallbackLink != null;
+  }
+
+  private INodeLink<T> getRootFallbackLink() {
+    Preconditions.checkState(root.isInternalDir());
+    return rootFallbackLink;
+  }
+
+  /**
+   * An internal class representing the ViewFileSystem mount table
+   * link entries and their attributes.
+   * @see LinkType
+   */
+  private static class LinkEntry {
+    private final String src;
+    private final String target;
+    private final LinkType linkType;
+    private final String settings;
+    private final UserGroupInformation ugi;
+    private final Configuration config;
+
+    LinkEntry(String src, String target, LinkType linkType, String settings,
+        UserGroupInformation ugi, Configuration config) {
+      this.src = src;
+      this.target = target;
+      this.linkType = linkType;
+      this.settings = settings;
+      this.ugi = ugi;
+      this.config = config;
+    }
+
+    String getSrc() {
+      return src;
+    }
+
+    String getTarget() {
+      return target;
+    }
+
+    LinkType getLinkType() {
+      return linkType;
+    }
+
+    boolean isLinkType(LinkType type) {
+      return this.linkType == type;
+    }
+
+    String getSettings() {
+      return settings;
+    }
+
+    UserGroupInformation getUgi() {
+      return ugi;
+    }
+
+    Configuration getConfig() {
+      return config;
+    }
+  }
+
   /**
    * Create Inode Tree from the specified mount-table specified in Config
    * @param config - the mount table keys are prefixed with 
@@ -286,39 +451,59 @@ abstract class InodeTree<T> {
   protected InodeTree(final Configuration config, final String viewName)
       throws UnsupportedFileSystemException, URISyntaxException,
       FileAlreadyExistsException, IOException {
-    String vName = viewName;
-    if (vName == null) {
-      vName = Constants.CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE;
+    String mountTableName = viewName;
+    if (mountTableName == null) {
+      mountTableName = Constants.CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE;
     }
-    homedirPrefix = ConfigUtil.getHomeDirValue(config, vName);
-    root = new INodeDir<T>("/", UserGroupInformation.getCurrentUser());
-    root.InodeDirFs = getTargetFileSystem(root);
-    root.isRoot = true;
+    homedirPrefix = ConfigUtil.getHomeDirValue(config, mountTableName);
+
+    boolean isMergeSlashConfigured = false;
+    String mergeSlashTarget = null;
+    List<LinkEntry> linkEntries = new LinkedList<>();
 
-    final String mtPrefix = Constants.CONFIG_VIEWFS_PREFIX + "." +
-        vName + ".";
+    final String mountTablePrefix =
+        Constants.CONFIG_VIEWFS_PREFIX + "." + mountTableName + ".";
     final String linkPrefix = Constants.CONFIG_VIEWFS_LINK + ".";
+    final String linkFallbackPrefix = Constants.CONFIG_VIEWFS_LINK_FALLBACK;
     final String linkMergePrefix = Constants.CONFIG_VIEWFS_LINK_MERGE + ".";
+    final String linkMergeSlashPrefix =
+        Constants.CONFIG_VIEWFS_LINK_MERGE_SLASH;
     boolean gotMountTableEntry = false;
     final UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
     for (Entry<String, String> si : config) {
       final String key = si.getKey();
-      if (key.startsWith(mtPrefix)) {
+      if (key.startsWith(mountTablePrefix)) {
         gotMountTableEntry = true;
-        LinkType linkType = LinkType.SINGLE;
-        String src = key.substring(mtPrefix.length());
+        LinkType linkType;
+        String src = key.substring(mountTablePrefix.length());
         String settings = null;
         if (src.startsWith(linkPrefix)) {
           src = src.substring(linkPrefix.length());
           if (src.equals(SlashPath.toString())) {
             throw new UnsupportedFileSystemException("Unexpected mount table "
-                + "link entry '" + key + "'. "
-                + Constants.CONFIG_VIEWFS_LINK_MERGE_SLASH  + " is not "
-                + "supported yet.");
+                + "link entry '" + key + "'. Use "
+                + Constants.CONFIG_VIEWFS_LINK_MERGE_SLASH  + " instead!");
           }
+          linkType = LinkType.SINGLE;
+        } else if (src.startsWith(linkFallbackPrefix)) {
+          if (src.length() != linkFallbackPrefix.length()) {
+            throw new IOException("ViewFs: Mount points initialization error." +
+                " Invalid " + Constants.CONFIG_VIEWFS_LINK_FALLBACK +
+                " entry in config: " + src);
+          }
+          linkType = LinkType.SINGLE_FALLBACK;
         } else if (src.startsWith(linkMergePrefix)) { // A merge link
-          linkType = LinkType.MERGE;
           src = src.substring(linkMergePrefix.length());
+          linkType = LinkType.MERGE;
+        } else if (src.startsWith(linkMergeSlashPrefix)) {
+          // This is a LinkMergeSlash entry. This entry should
+          // not have any additional source path.
+          if (src.length() != linkMergeSlashPrefix.length()) {
+            throw new IOException("ViewFs: Mount points initialization error." +
+                " Invalid " + Constants.CONFIG_VIEWFS_LINK_MERGE_SLASH +
+                " entry in config: " + src);
+          }
+          linkType = LinkType.MERGE_SLASH;
         } else if (src.startsWith(Constants.CONFIG_VIEWFS_LINK_NFLY)) {
           // prefix.settings.src
           src = src.substring(Constants.CONFIG_VIEWFS_LINK_NFLY.length() + 1);
@@ -338,14 +523,69 @@ abstract class InodeTree<T> {
           throw new IOException("ViewFs: Cannot initialize: Invalid entry in " +
               "Mount table in config: " + src);
         }
-        final String target = si.getValue(); // link or merge link
-        createLink(src, target, linkType, settings, ugi, config);
+
+        final String target = si.getValue();
+        if (linkType != LinkType.MERGE_SLASH) {
+          if (isMergeSlashConfigured) {
+            throw new IOException("Mount table " + mountTableName
+                + " has already been configured with a merge slash link. "
+                + "A regular link should not be added.");
+          }
+          linkEntries.add(
+              new LinkEntry(src, target, linkType, settings, ugi, config));
+        } else {
+          if (!linkEntries.isEmpty()) {
+            throw new IOException("Mount table " + mountTableName
+                + " has already been configured with regular links. "
+                + "A merge slash link should not be configured.");
+          }
+          if (isMergeSlashConfigured) {
+            throw new IOException("Mount table " + mountTableName
+                + " has already been configured with a merge slash link. "
+                + "Multiple merge slash links for the same mount table is "
+                + "not allowed.");
+          }
+          isMergeSlashConfigured = true;
+          mergeSlashTarget = target;
+        }
+      }
+    }
+
+    if (isMergeSlashConfigured) {
+      Preconditions.checkNotNull(mergeSlashTarget);
+      root = new INodeLink<T>(mountTableName, ugi,
+          getTargetFileSystem(new URI(mergeSlashTarget)),
+          new URI(mergeSlashTarget));
+      mountPoints.add(new MountPoint<T>("/", (INodeLink<T>) root));
+      rootFallbackLink = null;
+    } else {
+      root = new INodeDir<T>("/", UserGroupInformation.getCurrentUser());
+      getRootDir().setInternalDirFs(getTargetFileSystem(getRootDir()));
+      getRootDir().setRoot(true);
+      INodeLink<T> fallbackLink = null;
+      for (LinkEntry le : linkEntries) {
+        if (le.isLinkType(LinkType.SINGLE_FALLBACK)) {
+          if (fallbackLink != null) {
+            throw new IOException("Mount table " + mountTableName
+                + " has already been configured with a link fallback. "
+                + "Multiple fallback links for the same mount table is "
+                + "not allowed.");
+          }
+          fallbackLink = new INodeLink<T>(mountTableName, ugi,
+              getTargetFileSystem(new URI(le.getTarget())),
+              new URI(le.getTarget()));
+        } else {
+          createLink(le.getSrc(), le.getTarget(), le.getLinkType(),
+              le.getSettings(), le.getUgi(), le.getConfig());
+        }
       }
+      rootFallbackLink = fallbackLink;
     }
+
     if (!gotMountTableEntry) {
       throw new IOException(
           "ViewFs: Cannot initialize: Empty Mount table in config for " +
-              "viewfs://" + vName + "/");
+              "viewfs://" + mountTableName + "/");
     }
   }
 
@@ -382,7 +622,7 @@ abstract class InodeTree<T> {
 
   /**
    * Resolve the pathname p relative to root InodeDir
-   * @param p - inout path
+   * @param p - input path
    * @param resolveLastComponent
    * @return ResolveResult which allows further resolution of the remaining path
    * @throws FileNotFoundException
@@ -391,27 +631,53 @@ abstract class InodeTree<T> {
       throws FileNotFoundException {
     String[] path = breakIntoPathComponents(p);
     if (path.length <= 1) { // special case for when path is "/"
-      ResolveResult<T> res =
-          new ResolveResult<T>(ResultKind.INTERNAL_DIR,
-              root.InodeDirFs, root.fullPath, SlashPath);
+      T targetFs = root.isInternalDir() ?
+          getRootDir().getInternalDirFs() : getRootLink().getTargetFileSystem();
+      ResolveResult<T> res = new ResolveResult<T>(ResultKind.INTERNAL_DIR,
+          targetFs, root.fullPath, SlashPath);
       return res;
     }
 
-    INodeDir<T> curInode = root;
+    /**
+     * linkMergeSlash has been configured. The root of this mount table has
+     * been linked to the root directory of a file system.
+     * The first non-slash path component should be name of the mount table.
+     */
+    if (root.isLink()) {
+      Path remainingPath;
+      StringBuilder remainingPathStr = new StringBuilder();
+      // ignore first slash
+      for (int i = 1; i < path.length; i++) {
+        remainingPathStr.append("/").append(path[i]);
+      }
+      remainingPath = new Path(remainingPathStr.toString());
+      ResolveResult<T> res = new ResolveResult<T>(ResultKind.EXTERNAL_DIR,
+          getRootLink().getTargetFileSystem(), root.fullPath, remainingPath);
+      return res;
+    }
+    Preconditions.checkState(root.isInternalDir());
+    INodeDir<T> curInode = getRootDir();
+
     int i;
     // ignore first slash
     for (i = 1; i < path.length - (resolveLastComponent ? 0 : 1); i++) {
       INode<T> nextInode = curInode.resolveInternal(path[i]);
       if (nextInode == null) {
-        StringBuilder failedAt = new StringBuilder(path[0]);
-        for (int j = 1; j <= i; ++j) {
-          failedAt.append('/').append(path[j]);
+        if (hasFallbackLink()) {
+          return new ResolveResult<T>(ResultKind.EXTERNAL_DIR,
+              getRootFallbackLink().getTargetFileSystem(),
+              root.fullPath, new Path(p));
+        } else {
+          StringBuilder failedAt = new StringBuilder(path[0]);
+          for (int j = 1; j <= i; ++j) {
+            failedAt.append('/').append(path[j]);
+          }
+          throw (new FileNotFoundException(
+              "File/Directory does not exist: " + failedAt.toString()));
         }
-        throw (new FileNotFoundException("File/Directory does not exist: "
-            + failedAt.toString()));
       }
 
-      if (nextInode instanceof INodeLink) {
+      if (nextInode.isLink()) {
         final INodeLink<T> link = (INodeLink<T>) nextInode;
         final Path remainingPath;
         if (i >= path.length - 1) {
@@ -425,9 +691,9 @@ abstract class InodeTree<T> {
         }
         final ResolveResult<T> res =
             new ResolveResult<T>(ResultKind.EXTERNAL_DIR,
-                link.targetFileSystem, nextInode.fullPath, remainingPath);
+                link.getTargetFileSystem(), nextInode.fullPath, remainingPath);
         return res;
-      } else if (nextInode instanceof INodeDir) {
+      } else if (nextInode.isInternalDir()) {
         curInode = (INodeDir<T>) nextInode;
       }
     }
@@ -449,7 +715,7 @@ abstract class InodeTree<T> {
     }
     final ResolveResult<T> res =
         new ResolveResult<T>(ResultKind.INTERNAL_DIR,
-            curInode.InodeDirFs, curInode.fullPath, remainingPath);
+            curInode.getInternalDirFs(), curInode.fullPath, remainingPath);
     return res;
   }
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java

@@ -1037,12 +1037,12 @@ public class ViewFileSystem extends FileSystem {
     public FileStatus[] listStatus(Path f) throws AccessControlException,
         FileNotFoundException, IOException {
       checkPathIsSlash(f);
-      FileStatus[] result = new FileStatus[theInternalDir.children.size()];
+      FileStatus[] result = new FileStatus[theInternalDir.getChildren().size()];
       int i = 0;
-      for (Entry<String, INode<FileSystem>> iEntry : 
-                                          theInternalDir.children.entrySet()) {
+      for (Entry<String, INode<FileSystem>> iEntry :
+          theInternalDir.getChildren().entrySet()) {
         INode<FileSystem> inode = iEntry.getValue();
-        if (inode instanceof INodeLink ) {
+        if (inode.isLink()) {
           INodeLink<FileSystem> link = (INodeLink<FileSystem>) inode;
 
           result[i++] = new FileStatus(0, false, 0, 0,
@@ -1065,11 +1065,12 @@ public class ViewFileSystem extends FileSystem {
     @Override
     public boolean mkdirs(Path dir, FsPermission permission)
         throws AccessControlException, FileAlreadyExistsException {
-      if (theInternalDir.isRoot && dir == null) {
+      if (theInternalDir.isRoot() && dir == null) {
         throw new FileAlreadyExistsException("/ already exits");
       }
       // Note dir starts with /
-      if (theInternalDir.children.containsKey(dir.toString().substring(1))) {
+      if (theInternalDir.getChildren().containsKey(
+          dir.toString().substring(1))) {
         return true; // this is the stupid semantics of FileSystem
       }
       throw readOnlyMountTable("mkdirs",  dir);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java

@@ -899,13 +899,13 @@ public class ViewFs extends AbstractFileSystem {
         throws IOException {
       // look up i internalDirs children - ignore first Slash
       INode<AbstractFileSystem> inode =
-        theInternalDir.children.get(f.toUri().toString().substring(1)); 
+          theInternalDir.getChildren().get(f.toUri().toString().substring(1));
       if (inode == null) {
         throw new FileNotFoundException(
             "viewFs internal mount table - missing entry:" + f);
       }
       FileStatus result;
-      if (inode instanceof INodeLink) {
+      if (inode.isLink()) {
         INodeLink<AbstractFileSystem> inodelink = 
           (INodeLink<AbstractFileSystem>) inode;
         result = new FileStatus(0, false, 0, 0, creationTime, creationTime,
@@ -947,14 +947,14 @@ public class ViewFs extends AbstractFileSystem {
     public FileStatus[] listStatus(final Path f) throws AccessControlException,
         IOException {
       checkPathIsSlash(f);
-      FileStatus[] result = new FileStatus[theInternalDir.children.size()];
+      FileStatus[] result = new FileStatus[theInternalDir.getChildren().size()];
       int i = 0;
-      for (Entry<String, INode<AbstractFileSystem>> iEntry : 
-                                          theInternalDir.children.entrySet()) {
+      for (Entry<String, INode<AbstractFileSystem>> iEntry :
+          theInternalDir.getChildren().entrySet()) {
         INode<AbstractFileSystem> inode = iEntry.getValue();
 
         
-        if (inode instanceof INodeLink ) {
+        if (inode.isLink()) {
           INodeLink<AbstractFileSystem> link = 
             (INodeLink<AbstractFileSystem>) inode;
 
@@ -979,7 +979,7 @@ public class ViewFs extends AbstractFileSystem {
     public void mkdir(final Path dir, final FsPermission permission,
         final boolean createParent) throws AccessControlException,
         FileAlreadyExistsException {
-      if (theInternalDir.isRoot && dir == null) {
+      if (theInternalDir.isRoot() && dir == null) {
         throw new FileAlreadyExistsException("/ already exits");
       }
       throw readOnlyMountTable("mkdir", dir);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java

@@ -184,40 +184,45 @@ public abstract class ZKFailoverController {
         }
       });
     } catch (RuntimeException rte) {
-      LOG.error("The failover controller encounters runtime error: " + rte);
       throw (Exception)rte.getCause();
     }
   }
   
 
   private int doRun(String[] args)
-      throws HadoopIllegalArgumentException, IOException, InterruptedException {
+      throws Exception {
     try {
       initZK();
     } catch (KeeperException ke) {
       LOG.error("Unable to start failover controller. Unable to connect "
           + "to ZooKeeper quorum at " + zkQuorum + ". Please check the "
           + "configured value for " + ZK_QUORUM_KEY + " and ensure that "
-          + "ZooKeeper is running.");
+          + "ZooKeeper is running.", ke);
       return ERR_CODE_NO_ZK;
     }
-    if (args.length > 0) {
-      if ("-formatZK".equals(args[0])) {
-        boolean force = false;
-        boolean interactive = true;
-        for (int i = 1; i < args.length; i++) {
-          if ("-force".equals(args[i])) {
-            force = true;
-          } else if ("-nonInteractive".equals(args[i])) {
-            interactive = false;
-          } else {
-            badArg(args[i]);
+    try {
+      if (args.length > 0) {
+        if ("-formatZK".equals(args[0])) {
+          boolean force = false;
+          boolean interactive = true;
+          for (int i = 1; i < args.length; i++) {
+            if ("-force".equals(args[i])) {
+              force = true;
+            } else if ("-nonInteractive".equals(args[i])) {
+              interactive = false;
+            } else {
+              badArg(args[i]);
+            }
           }
+          return formatZK(force, interactive);
+        }
+        else {
+          badArg(args[0]);
         }
-        return formatZK(force, interactive);
-      } else {
-        badArg(args[0]);
       }
+    } catch (Exception e){
+      LOG.error("The failover controller encounters runtime error", e);
+      throw e;
     }
 
     if (!elector.parentZNodeExists()) {
@@ -236,11 +241,14 @@ public abstract class ZKFailoverController {
       return ERR_CODE_NO_FENCER;
     }
 
-    initRPC();
-    initHM();
-    startRPC();
     try {
+      initRPC();
+      initHM();
+      startRPC();
       mainLoop();
+    } catch (Exception e) {
+      LOG.error("The failover controller encounters runtime error: ", e);
+      throw e;
     } finally {
       rpcServer.stopAndJoin();
       

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java

@@ -50,6 +50,7 @@ public final class ErasureCodeConstants {
   public static final ECSchema REPLICATION_1_2_SCHEMA = new ECSchema(
       REPLICATION_CODEC_NAME, 1, 2);
 
+  public static final byte MAX_POLICY_ID = Byte.MAX_VALUE;
   public static final byte USER_DEFINED_POLICY_START_ID = (byte) 64;
   public static final byte REPLICATION_POLICY_ID = (byte) 63;
   public static final String REPLICATION_POLICY_NAME = REPLICATION_CODEC_NAME;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java

@@ -21,6 +21,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.erasurecode.ECBlock;
 import org.apache.hadoop.io.erasurecode.ECChunk;
 
+import java.io.IOException;
+
 /**
  * Erasure coding step that's involved in encoding/decoding of a block group.
  */
@@ -47,7 +49,8 @@ public interface ErasureCodingStep {
    * @param inputChunks
    * @param outputChunks
    */
-  void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks);
+  void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks)
+      throws IOException;
 
   /**
    * Notify erasure coder that all the chunks of input blocks are processed so

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java

@@ -22,6 +22,8 @@ import org.apache.hadoop.io.erasurecode.ECBlock;
 import org.apache.hadoop.io.erasurecode.ECChunk;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
 
+import java.io.IOException;
+
 /**
  * Erasure decoding step, a wrapper of all the necessary information to perform
  * a decoding step involved in the whole process of decoding a block group.
@@ -50,7 +52,8 @@ public class ErasureDecodingStep implements ErasureCodingStep {
   }
 
   @Override
-  public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks) {
+  public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks)
+      throws IOException {
     rawDecoder.decode(inputChunks, erasedIndexes, outputChunks);
   }
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java

@@ -22,6 +22,8 @@ import org.apache.hadoop.io.erasurecode.ECBlock;
 import org.apache.hadoop.io.erasurecode.ECChunk;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
 
+import java.io.IOException;
+
 /**
  * Erasure encoding step, a wrapper of all the necessary information to perform
  * an encoding step involved in the whole process of encoding a block group.
@@ -46,7 +48,8 @@ public class ErasureEncodingStep implements ErasureCodingStep {
   }
 
   @Override
-  public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks) {
+  public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks)
+      throws IOException {
     rawEncoder.encode(inputChunks, outputChunks);
   }
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.io.erasurecode.coder;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -64,7 +65,8 @@ public class HHXORErasureDecodingStep extends HHErasureCodingStep {
   }
 
   @Override
-  public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks) {
+  public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks)
+      throws IOException {
     if (erasedIndexes.length == 0) {
       return;
     }
@@ -74,7 +76,8 @@ public class HHXORErasureDecodingStep extends HHErasureCodingStep {
     performCoding(inputBuffers, outputBuffers);
   }
 
-  private void performCoding(ByteBuffer[] inputs, ByteBuffer[] outputs) {
+  private void performCoding(ByteBuffer[] inputs, ByteBuffer[] outputs)
+      throws IOException {
     final int numDataUnits = rsRawDecoder.getNumDataUnits();
     final int numParityUnits = rsRawDecoder.getNumParityUnits();
     final int numTotalUnits = numDataUnits + numParityUnits;
@@ -119,7 +122,7 @@ public class HHXORErasureDecodingStep extends HHErasureCodingStep {
 
   private void doDecodeSingle(ByteBuffer[][] inputs, ByteBuffer[][] outputs,
                               int erasedLocationToFix, int bufSize,
-                              boolean isDirect) {
+                              boolean isDirect) throws IOException {
     final int numDataUnits = rsRawDecoder.getNumDataUnits();
     final int numParityUnits = rsRawDecoder.getNumParityUnits();
     final int subPacketSize = getSubPacketSize();
@@ -261,7 +264,8 @@ public class HHXORErasureDecodingStep extends HHErasureCodingStep {
 
   private void doDecodeMultiAndParity(ByteBuffer[][] inputs,
                                       ByteBuffer[][] outputs,
-                                      int[] erasedLocationToFix, int bufSize) {
+                                      int[] erasedLocationToFix, int bufSize)
+      throws IOException {
     final int numDataUnits = rsRawDecoder.getNumDataUnits();
     final int numParityUnits = rsRawDecoder.getNumParityUnits();
     final int numTotalUnits = numDataUnits + numParityUnits;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.io.erasurecode.coder;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -56,13 +57,15 @@ public class HHXORErasureEncodingStep extends HHErasureCodingStep {
   }
 
   @Override
-  public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks) {
+  public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks)
+      throws IOException {
     ByteBuffer[] inputBuffers = ECChunk.toBuffers(inputChunks);
     ByteBuffer[] outputBuffers = ECChunk.toBuffers(outputChunks);
     performCoding(inputBuffers, outputBuffers);
   }
 
-  private void performCoding(ByteBuffer[] inputs, ByteBuffer[] outputs) {
+  private void performCoding(ByteBuffer[] inputs, ByteBuffer[] outputs)
+      throws IOException {
     final int numDataUnits = this.rsRawEncoder.getNumDataUnits();
     final int numParityUnits = this.rsRawEncoder.getNumParityUnits();
     final int subSPacketSize = getSubPacketSize();
@@ -95,7 +98,8 @@ public class HHXORErasureEncodingStep extends HHErasureCodingStep {
     doEncode(hhInputs, hhOutputs);
   }
 
-  private void doEncode(ByteBuffer[][] inputs, ByteBuffer[][] outputs) {
+  private void doEncode(ByteBuffer[][] inputs, ByteBuffer[][] outputs)
+      throws IOException {
     final int numParityUnits = this.rsRawEncoder.getNumParityUnits();
 
     // calc piggyBacks using first sub-packet

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/util/HHUtil.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.io.erasurecode.coder.util;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -64,7 +65,8 @@ public final class HHUtil {
                                                     int[] piggyBackIndex,
                                                     int numParityUnits,
                                                     int pgIndex,
-                                                    RawErasureEncoder encoder) {
+                                                    RawErasureEncoder encoder)
+      throws IOException {
     ByteBuffer[] emptyInput = new ByteBuffer[inputs.length];
     ByteBuffer[] tempInput = new ByteBuffer[inputs.length];
     int[] inputPositions = new int[inputs.length];

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java

@@ -23,6 +23,7 @@ import org.apache.hadoop.util.PerformanceAdvisory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 /**
@@ -38,7 +39,12 @@ abstract class AbstractNativeRawDecoder extends RawErasureDecoder {
   }
 
   @Override
-  protected void doDecode(ByteBufferDecodingState decodingState) {
+  protected synchronized void doDecode(ByteBufferDecodingState decodingState)
+      throws IOException {
+    if (nativeCoder == 0) {
+      throw new IOException(String.format("%s closed",
+          getClass().getSimpleName()));
+    }
     int[] inputOffsets = new int[decodingState.inputs.length];
     int[] outputOffsets = new int[decodingState.outputs.length];
 
@@ -63,10 +69,12 @@ abstract class AbstractNativeRawDecoder extends RawErasureDecoder {
   protected abstract void performDecodeImpl(ByteBuffer[] inputs,
                                             int[] inputOffsets, int dataLen,
                                             int[] erased, ByteBuffer[] outputs,
-                                            int[] outputOffsets);
+                                            int[] outputOffsets)
+      throws IOException;
 
   @Override
-  protected void doDecode(ByteArrayDecodingState decodingState) {
+  protected void doDecode(ByteArrayDecodingState decodingState)
+      throws IOException {
     PerformanceAdvisory.LOG.debug("convertToByteBufferState is invoked, " +
         "not efficiently. Please use direct ByteBuffer inputs/outputs");
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java

@@ -23,6 +23,7 @@ import org.apache.hadoop.util.PerformanceAdvisory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 /**
@@ -38,7 +39,12 @@ abstract class AbstractNativeRawEncoder extends RawErasureEncoder {
   }
 
   @Override
-  protected void doEncode(ByteBufferEncodingState encodingState) {
+  protected synchronized void doEncode(ByteBufferEncodingState encodingState)
+      throws IOException {
+    if (nativeCoder == 0) {
+      throw new IOException(String.format("%s closed",
+          getClass().getSimpleName()));
+    }
     int[] inputOffsets = new int[encodingState.inputs.length];
     int[] outputOffsets = new int[encodingState.outputs.length];
     int dataLen = encodingState.inputs[0].remaining();
@@ -60,10 +66,12 @@ abstract class AbstractNativeRawEncoder extends RawErasureEncoder {
 
   protected abstract void performEncodeImpl(
           ByteBuffer[] inputs, int[] inputOffsets,
-          int dataLen, ByteBuffer[] outputs, int[] outputOffsets);
+          int dataLen, ByteBuffer[] outputs, int[] outputOffsets)
+      throws IOException;
 
   @Override
-  protected void doEncode(ByteArrayEncodingState encodingState) {
+  protected void doEncode(ByteArrayEncodingState encodingState)
+      throws IOException {
     PerformanceAdvisory.LOG.debug("convertToByteBufferState is invoked, " +
         "not efficiently. Please use direct ByteBuffer inputs/outputs");
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java

@@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 /**
@@ -39,14 +40,14 @@ public class NativeRSRawDecoder extends AbstractNativeRawDecoder {
   }
 
   @Override
-  protected void performDecodeImpl(ByteBuffer[] inputs, int[] inputOffsets,
-                                   int dataLen, int[] erased,
-                                   ByteBuffer[] outputs, int[] outputOffsets) {
+  protected synchronized void performDecodeImpl(
+      ByteBuffer[] inputs, int[] inputOffsets, int dataLen, int[] erased,
+      ByteBuffer[] outputs, int[] outputOffsets) throws IOException {
     decodeImpl(inputs, inputOffsets, dataLen, erased, outputs, outputOffsets);
   }
 
   @Override
-  public void release() {
+  public synchronized void release() {
     destroyImpl();
   }
 
@@ -59,7 +60,7 @@ public class NativeRSRawDecoder extends AbstractNativeRawDecoder {
 
   private native void decodeImpl(
           ByteBuffer[] inputs, int[] inputOffsets, int dataLen, int[] erased,
-          ByteBuffer[] outputs, int[] outputOffsets);
+          ByteBuffer[] outputs, int[] outputOffsets) throws IOException;
 
   private native void destroyImpl();
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java

@@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 /**
@@ -39,14 +40,14 @@ public class NativeRSRawEncoder extends AbstractNativeRawEncoder {
   }
 
   @Override
-  protected void performEncodeImpl(
+  protected synchronized void performEncodeImpl(
           ByteBuffer[] inputs, int[] inputOffsets, int dataLen,
-          ByteBuffer[] outputs, int[] outputOffsets) {
+          ByteBuffer[] outputs, int[] outputOffsets) throws IOException {
     encodeImpl(inputs, inputOffsets, dataLen, outputs, outputOffsets);
   }
 
   @Override
-  public void release() {
+  public synchronized void release() {
     destroyImpl();
   }
 
@@ -58,8 +59,8 @@ public class NativeRSRawEncoder extends AbstractNativeRawEncoder {
   private native void initImpl(int numDataUnits, int numParityUnits);
 
   private native void encodeImpl(ByteBuffer[] inputs, int[] inputOffsets,
-                                        int dataLen, ByteBuffer[] outputs,
-                                        int[] outputOffsets);
+                                 int dataLen, ByteBuffer[] outputs,
+                                 int[] outputOffsets) throws IOException;
 
   private native void destroyImpl();
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java

@@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 /**
@@ -39,21 +40,26 @@ public class NativeXORRawDecoder extends AbstractNativeRawDecoder {
   }
 
   @Override
-  protected void performDecodeImpl(ByteBuffer[] inputs, int[] inputOffsets,
-      int dataLen, int[] erased, ByteBuffer[] outputs, int[] outputOffsets) {
+  protected synchronized void performDecodeImpl(
+      ByteBuffer[] inputs, int[] inputOffsets, int dataLen, int[] erased,
+      ByteBuffer[] outputs, int[] outputOffsets) throws IOException {
     decodeImpl(inputs, inputOffsets, dataLen, erased, outputs, outputOffsets);
   }
 
   @Override
-  public void release() {
+  public synchronized void release() {
     destroyImpl();
   }
 
   private native void initImpl(int numDataUnits, int numParityUnits);
 
+  /**
+   * Native implementation of decoding.
+   * @throws IOException if the decoder is closed.
+   */
   private native void decodeImpl(
       ByteBuffer[] inputs, int[] inputOffsets, int dataLen, int[] erased,
-      ByteBuffer[] outputs, int[] outputOffsets);
+      ByteBuffer[] outputs, int[] outputOffsets) throws IOException;
 
   private native void destroyImpl();
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java

@@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 /**
@@ -39,14 +40,14 @@ public class NativeXORRawEncoder extends AbstractNativeRawEncoder {
   }
 
   @Override
-  protected void performEncodeImpl(
+  protected synchronized void performEncodeImpl(
       ByteBuffer[] inputs, int[] inputOffsets, int dataLen,
-      ByteBuffer[] outputs, int[] outputOffsets) {
+      ByteBuffer[] outputs, int[] outputOffsets) throws IOException {
     encodeImpl(inputs, inputOffsets, dataLen, outputs, outputOffsets);
   }
 
   @Override
-  public void release() {
+  public synchronized void release() {
     destroyImpl();
   }
 
@@ -54,7 +55,7 @@ public class NativeXORRawEncoder extends AbstractNativeRawEncoder {
 
   private native void encodeImpl(ByteBuffer[] inputs, int[] inputOffsets,
                                  int dataLen, ByteBuffer[] outputs,
-                                 int[] outputOffsets);
+                                 int[] outputOffsets) throws IOException;
 
   private native void destroyImpl();
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawDecoder.java

@@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 import org.apache.hadoop.io.erasurecode.rawcoder.util.RSUtil;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 /**
@@ -53,7 +54,7 @@ public class RSLegacyRawDecoder extends RawErasureDecoder {
 
   @Override
   public void decode(ByteBuffer[] inputs, int[] erasedIndexes,
-                     ByteBuffer[] outputs) {
+                     ByteBuffer[] outputs) throws IOException {
     // Make copies avoiding affecting original ones;
     ByteBuffer[] newInputs = new ByteBuffer[inputs.length];
     int[] newErasedIndexes = new int[erasedIndexes.length];
@@ -67,7 +68,8 @@ public class RSLegacyRawDecoder extends RawErasureDecoder {
   }
 
   @Override
-  public void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs) {
+  public void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs)
+      throws IOException {
     // Make copies avoiding affecting original ones;
     byte[][] newInputs = new byte[inputs.length][];
     int[] newErasedIndexes = new int[erasedIndexes.length];

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java

@@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.erasurecode.ECChunk;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 /**
@@ -81,7 +82,7 @@ public abstract class RawErasureDecoder {
    *                erasedIndexes, ready for read after the call
    */
   public void decode(ByteBuffer[] inputs, int[] erasedIndexes,
-                     ByteBuffer[] outputs) {
+                     ByteBuffer[] outputs) throws IOException {
     ByteBufferDecodingState decodingState = new ByteBufferDecodingState(this,
         inputs, erasedIndexes, outputs);
 
@@ -117,7 +118,8 @@ public abstract class RawErasureDecoder {
    * Perform the real decoding using Direct ByteBuffer.
    * @param decodingState the decoding state
    */
-  protected abstract void doDecode(ByteBufferDecodingState decodingState);
+  protected abstract void doDecode(ByteBufferDecodingState decodingState)
+      throws IOException;
 
   /**
    * Decode with inputs and erasedIndexes, generates outputs. More see above.
@@ -126,8 +128,10 @@ public abstract class RawErasureDecoder {
    * @param erasedIndexes indexes of erased units in the inputs array
    * @param outputs output buffers to put decoded data into according to
    *                erasedIndexes, ready for read after the call
+   * @throws IOException if the decoder is closed.
    */
-  public void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs) {
+  public void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs)
+      throws IOException {
     ByteArrayDecodingState decodingState = new ByteArrayDecodingState(this,
         inputs, erasedIndexes, outputs);
 
@@ -142,8 +146,10 @@ public abstract class RawErasureDecoder {
    * Perform the real decoding using bytes array, supporting offsets and
    * lengths.
    * @param decodingState the decoding state
+   * @throws IOException if the decoder is closed.
    */
-  protected abstract void doDecode(ByteArrayDecodingState decodingState);
+  protected abstract void doDecode(ByteArrayDecodingState decodingState)
+      throws IOException;
 
   /**
    * Decode with inputs and erasedIndexes, generates outputs. More see above.
@@ -155,9 +161,10 @@ public abstract class RawErasureDecoder {
    * @param erasedIndexes indexes of erased units in the inputs array
    * @param outputs output buffers to put decoded data into according to
    *                erasedIndexes, ready for read after the call
+   * @throws IOException if the decoder is closed
    */
   public void decode(ECChunk[] inputs, int[] erasedIndexes,
-                     ECChunk[] outputs) {
+                     ECChunk[] outputs) throws IOException {
     ByteBuffer[] newInputs = CoderUtil.toBuffers(inputs);
     ByteBuffer[] newOutputs = CoderUtil.toBuffers(outputs);
     decode(newInputs, erasedIndexes, newOutputs);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java

@@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.erasurecode.ECChunk;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 /**
@@ -62,8 +63,10 @@ public abstract class RawErasureEncoder {
    *               be 0 after encoding
    * @param outputs output buffers to put the encoded data into, ready to read
    *                after the call
+   * @throws IOException if the encoder is closed.
    */
-  public void encode(ByteBuffer[] inputs, ByteBuffer[] outputs) {
+  public void encode(ByteBuffer[] inputs, ByteBuffer[] outputs)
+      throws IOException {
     ByteBufferEncodingState bbeState = new ByteBufferEncodingState(
         this, inputs, outputs);
 
@@ -99,7 +102,8 @@ public abstract class RawErasureEncoder {
    * Perform the real encoding work using direct ByteBuffer.
    * @param encodingState the encoding state
    */
-  protected abstract void doEncode(ByteBufferEncodingState encodingState);
+  protected abstract void doEncode(ByteBufferEncodingState encodingState)
+      throws IOException;
 
   /**
    * Encode with inputs and generates outputs. More see above.
@@ -108,7 +112,7 @@ public abstract class RawErasureEncoder {
    * @param outputs output buffers to put the encoded data into, read to read
    *                after the call
    */
-  public void encode(byte[][] inputs, byte[][] outputs) {
+  public void encode(byte[][] inputs, byte[][] outputs) throws IOException {
     ByteArrayEncodingState baeState = new ByteArrayEncodingState(
         this, inputs, outputs);
 
@@ -125,7 +129,8 @@ public abstract class RawErasureEncoder {
    * and lengths.
    * @param encodingState the encoding state
    */
-  protected abstract void doEncode(ByteArrayEncodingState encodingState);
+  protected abstract void doEncode(ByteArrayEncodingState encodingState)
+      throws IOException;
 
   /**
    * Encode with inputs and generates outputs. More see above.
@@ -133,8 +138,9 @@ public abstract class RawErasureEncoder {
    * @param inputs input buffers to read data from
    * @param outputs output buffers to put the encoded data into, read to read
    *                after the call
+   * @throws IOException if the encoder is closed.
    */
-  public void encode(ECChunk[] inputs, ECChunk[] outputs) {
+  public void encode(ECChunk[] inputs, ECChunk[] outputs) throws IOException {
     ByteBuffer[] newInputs = ECChunk.toBuffers(inputs);
     ByteBuffer[] newOutputs = ECChunk.toBuffers(outputs);
     encode(newInputs, newOutputs);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java

@@ -212,7 +212,8 @@ public class Client implements AutoCloseable {
    * @param conf Configuration
    * @param pingInterval the ping interval
    */
-  static final void setPingInterval(Configuration conf, int pingInterval) {
+  public static final void setPingInterval(Configuration conf,
+      int pingInterval) {
     conf.setInt(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY, pingInterval);
   }
 
@@ -223,7 +224,7 @@ public class Client implements AutoCloseable {
    * @param conf Configuration
    * @return the ping interval
    */
-  static final int getPingInterval(Configuration conf) {
+  public static final int getPingInterval(Configuration conf) {
     return conf.getInt(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY,
         CommonConfigurationKeys.IPC_PING_INTERVAL_DEFAULT);
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java

@@ -58,6 +58,11 @@ public class JvmMetrics implements MetricsSource {
       }
       return impl;
     }
+
+    synchronized void shutdown() {
+      DefaultMetricsSystem.instance().unregisterSource(JvmMetrics.name());
+      impl = null;
+    }
   }
 
   @VisibleForTesting
@@ -81,6 +86,7 @@ public class JvmMetrics implements MetricsSource {
   final ConcurrentHashMap<String, MetricsInfo[]> gcInfoCache =
       new ConcurrentHashMap<String, MetricsInfo[]>();
 
+  @VisibleForTesting
   JvmMetrics(String processName, String sessionId) {
     this.processName = processName;
     this.sessionId = sessionId;
@@ -104,6 +110,16 @@ public class JvmMetrics implements MetricsSource {
     return Singleton.INSTANCE.init(processName, sessionId);
   }
 
+  /**
+   * Shutdown the JvmMetrics singleton. This is not necessary if the JVM itself
+   * is shutdown, but may be necessary for scenarios where JvmMetrics instance
+   * needs to be re-created while the JVM is still around. One such scenario
+   * is unit-testing.
+   */
+  public static void shutdownSingleton() {
+    Singleton.INSTANCE.shutdown();
+  }
+
   @Override
   public void getMetrics(MetricsCollector collector, boolean all) {
     MetricsRecordBuilder rb = collector.addRecord(JvmMetrics)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/ExecutorHelper.java

@@ -40,14 +40,16 @@ public final class ExecutorHelper {
 
     //For additional information, see: https://docs.oracle
     // .com/javase/7/docs/api/java/util/concurrent/ThreadPoolExecutor
-    // .html#afterExecute(java.lang.Runnable,%20java.lang.Throwable) .
+    // .html#afterExecute(java.lang.Runnable,%20java.lang.Throwable)
 
-    if (t == null && r instanceof Future<?>) {
+    // Handle JDK-8071638
+    if (t == null && r instanceof Future<?> && ((Future<?>) r).isDone()) {
       try {
         ((Future<?>) r).get();
       } catch (ExecutionException ee) {
-        LOG.warn("Execution exception when running task in " +
-            Thread.currentThread().getName());
+        LOG.warn(
+            "Execution exception when running task in " + Thread.currentThread()
+                .getName());
         t = ee.getCause();
       } catch (InterruptedException ie) {
         LOG.warn("Thread (" + Thread.currentThread() + ") interrupted: ", ie);

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c

@@ -63,8 +63,9 @@ IsalCoder* getCoder(JNIEnv* env, jobject thiz) {
                                     "Field nativeCoder not found");
   }
   pCoder = (IsalCoder*)(*env)->GetLongField(env, thiz, fid);
-  pCoder->verbose = (verbose == JNI_TRUE) ? 1 : 0;
-
+  if (pCoder != NULL) {
+    pCoder->verbose = (verbose == JNI_TRUE) ? 1 : 0;
+  }
   return pCoder;
 }
 

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c

@@ -48,6 +48,10 @@ JNIEnv *env, jobject thiz, jobjectArray inputs, jintArray inputOffsets,
 jint dataLen, jintArray erasedIndexes, jobjectArray outputs,
 jintArray outputOffsets) {
   RSDecoder* rsDecoder = (RSDecoder*)getCoder(env, thiz);
+  if (!rsDecoder) {
+    THROW(env, "java/io/IOException", "NativeRSRawDecoder closed");
+    return;
+  }
 
   int numDataUnits = rsDecoder->decoder.coder.numDataUnits;
   int numParityUnits = rsDecoder->decoder.coder.numParityUnits;
@@ -68,5 +72,8 @@ JNIEXPORT void JNICALL
 Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_destroyImpl(
 JNIEnv *env, jobject thiz) {
   RSDecoder* rsDecoder = (RSDecoder*)getCoder(env, thiz);
-  free(rsDecoder);
+  if (rsDecoder) {
+    free(rsDecoder);
+    setCoder(env, thiz, NULL);
+  }
 }

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_encoder.c

@@ -47,6 +47,10 @@ Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_encodeImpl(
 JNIEnv *env, jobject thiz, jobjectArray inputs, jintArray inputOffsets,
 jint dataLen, jobjectArray outputs, jintArray outputOffsets) {
   RSEncoder* rsEncoder = (RSEncoder*)getCoder(env, thiz);
+  if (!rsEncoder) {
+    THROW(env, "java/io/IOException", "NativeRSRawEncoder closed");
+    return;
+  }
 
   int numDataUnits = rsEncoder->encoder.coder.numDataUnits;
   int numParityUnits = rsEncoder->encoder.coder.numParityUnits;
@@ -62,5 +66,8 @@ JNIEXPORT void JNICALL
 Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_destroyImpl(
 JNIEnv *env, jobject thiz) {
   RSEncoder* rsEncoder = (RSEncoder*)getCoder(env, thiz);
-  free(rsEncoder);
+  if (rsEncoder) {
+    free(rsEncoder);
+    setCoder(env, thiz, NULL);
+  }
 }

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c

@@ -54,6 +54,10 @@ Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawDecoder_decodeImpl(
   XORDecoder* xorDecoder;
 
   xorDecoder = (XORDecoder*)getCoder(env, thiz);
+  if (!xorDecoder) {
+    THROW(env, "java/io/IOException", "NativeXORRawDecoder closed");
+    return;
+  }
   numDataUnits = ((IsalCoder*)xorDecoder)->numDataUnits;
   numParityUnits = ((IsalCoder*)xorDecoder)->numParityUnits;
   chunkSize = (int)dataLen;
@@ -76,5 +80,8 @@ JNIEXPORT void JNICALL
 Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawDecoder_destroyImpl
   (JNIEnv *env, jobject thiz){
   XORDecoder* xorDecoder = (XORDecoder*)getCoder(env, thiz);
-  free(xorDecoder);
+  if (xorDecoder) {
+    free(xorDecoder);
+    setCoder(env, thiz, NULL);
+  }
 }

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c

@@ -54,6 +54,10 @@ Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawEncoder_encodeImpl(
   XOREncoder* xorEncoder;
 
   xorEncoder = (XOREncoder*)getCoder(env, thiz);
+  if (!xorEncoder) {
+    THROW(env, "java/io/IOException", "NativeXORRawEncoder closed");
+    return;
+  }
   numDataUnits = ((IsalCoder*)xorEncoder)->numDataUnits;
   numParityUnits = ((IsalCoder*)xorEncoder)->numParityUnits;
   chunkSize = (int)dataLen;
@@ -78,5 +82,8 @@ JNIEXPORT void JNICALL
 Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawEncoder_destroyImpl
   (JNIEnv *env, jobject thiz) {
   XOREncoder* xorEncoder = (XOREncoder*)getCoder(env, thiz);
-  free(xorEncoder);
+  if (xorEncoder) {
+    free(xorEncoder);
+    setCoder(env, thiz, NULL);
+  }
 }

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -2353,6 +2353,14 @@
     key will be dropped. Default = 12hrs
   </description>
 </property>
+<property>
+  <name>hadoop.security.kms.client.timeout</name>
+  <value>60</value>
+  <description>
+    Sets value for KMS client connection timeout, and the read timeout
+    to KMS servers.
+  </description>
+</property>
 
 <property>
   <name>hadoop.security.kms.client.failover.sleep.base.millis</name>
@@ -2776,4 +2784,17 @@
         the ZK CLI).
     </description>
   </property>
+  <property>
+    <name>hadoop.treat.subject.external</name>
+    <value>false</value>
+    <description>
+      When creating UGI with UserGroupInformation(Subject), treat the passed
+      subject external if set to true, and assume the owner of the subject
+      should do the credential renewal.
+
+      When true this property will introduce an incompatible change which
+      may require changes in client code. For more details, see the jiras:
+      HADOOP-13805,HADOOP-13558.
+    </description>
+  </property>
 </configuration>

hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md

@@ -85,9 +85,10 @@ This file should be readable only by the Unix user running the daemons.
 
 It is possible to set a maximum time limit when searching and awaiting a result.
 Set `hadoop.security.group.mapping.ldap.directory.search.timeout` to 0 if infinite wait period is desired. Default is 10,000 milliseconds (10 seconds).
+This is the limit for each ldap query.  If `hadoop.security.group.mapping.ldap.search.group.hierarchy.levels` is set to a positive value, then the total latency will be bounded by max(Recur Depth in LDAP, `hadoop.security.group.mapping.ldap.search.group.hierarchy.levels` ) * `hadoop.security.group.mapping.ldap.directory.search.timeout`.
 
-The implementation does not attempt to resolve group hierarchies. Therefore, a user must be an explicit member of a group object
-in order to be considered a member.
+`hadoop.security.group.mapping.ldap.base` configures how far to walk up the groups hierarchy when resolving groups.
+By default, with a limit of 0, in order to be considered a member of a group, the user must be an explicit member in LDAP.  Otherwise, it will traverse the group hierarchy `hadoop.security.group.mapping.ldap.search.group.hierarchy.levels` levels up.
 
 
 ### Active Directory ###

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java

@@ -2242,6 +2242,21 @@ public class TestConfiguration {
     FileUtil.fullyDelete(tmpDir);
   }
 
+  public void testGettingPropertiesWithPrefix() throws Exception {
+    Configuration conf = new Configuration();
+    for (int i = 0; i < 10; i++) {
+      conf.set("prefix" + ".name" + i, "value");
+    }
+    conf.set("different.prefix" + ".name", "value");
+    Map<String, String> props = conf.getPropsWithPrefix("prefix");
+    assertEquals(props.size(), 10);
+
+    // test call with no properties for a given prefix
+    props = conf.getPropsWithPrefix("none");
+    assertNotNull(props.isEmpty());
+    assertTrue(props.isEmpty());
+  }
+
   public static void main(String[] argv) throws Exception {
     junit.textui.TestRunner.main(new String[]{
       TestConfiguration.class.getName()

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java

@@ -670,6 +670,27 @@ public class ContractTestUtils extends Assert {
     assertPathDoesNotExist(fs, "Deleted file", file);
   }
 
+  /**
+   * Execute a {@link FileSystem#rename(Path, Path)}, and verify that the
+   * outcome was as expected. There is no preflight checking of arguments;
+   * everything is left to the rename() command.
+   * @param fs filesystem
+   * @param source source path
+   * @param dest destination path
+   * @param expectedResult expected return code
+   * @throws IOException on any IO failure.
+   */
+  public static void assertRenameOutcome(FileSystem fs,
+      Path source,
+      Path dest,
+      boolean expectedResult) throws IOException {
+    boolean result = fs.rename(source, dest);
+    if (expectedResult != result) {
+      fail(String.format("Expected rename(%s, %s) to return %b,"
+              + " but result was %b", source, dest, expectedResult, result));
+    }
+  }
+
   /**
    * Read in "length" bytes, convert to an ascii string.
    * @param fs filesystem

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java

@@ -1005,8 +1005,8 @@ abstract public class ViewFileSystemBaseTest {
           + mtPrefix + Constants.CONFIG_VIEWFS_LINK + "." + "/");
     } catch (Exception e) {
       if (e instanceof UnsupportedFileSystemException) {
-        String msg = Constants.CONFIG_VIEWFS_LINK_MERGE_SLASH
-            + " is not supported yet.";
+        String msg = " Use " + Constants.CONFIG_VIEWFS_LINK_MERGE_SLASH +
+            " instead";
         assertThat(e.getMessage(), containsString(msg));
       } else {
         fail("Unexpected exception: " + e.getMessage());

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java

@@ -23,8 +23,11 @@ import org.apache.hadoop.io.erasurecode.ECChunk;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 import org.apache.hadoop.io.erasurecode.TestCoderBase;
 
+import java.io.IOException;
 import java.lang.reflect.Constructor;
 
+import static org.junit.Assert.fail;
+
 /**
  * Erasure coder test base with utilities.
  */
@@ -85,14 +88,22 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
 
     ErasureCodingStep codingStep;
     codingStep = encoder.calculateCoding(blockGroup);
-    performCodingStep(codingStep);
+    try {
+      performCodingStep(codingStep);
+    } catch (IOException e) {
+      fail("Should not expect IOException: " + e.getMessage());
+    }
     // Erase specified sources but return copies of them for later comparing
     TestBlock[] backupBlocks = backupAndEraseBlocks(clonedDataBlocks, parityBlocks);
 
     // Decode
     blockGroup = new ECBlockGroup(clonedDataBlocks, blockGroup.getParityBlocks());
     codingStep = decoder.calculateCoding(blockGroup);
-    performCodingStep(codingStep);
+    try {
+      performCodingStep(codingStep);
+    } catch (IOException e) {
+      fail("Should not expect IOException: " + e.getMessage());
+    }
 
     // Compare
     compareAndVerify(backupBlocks, codingStep.getOutputBlocks());
@@ -102,7 +113,8 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
    * This is typically how a coding step should be performed.
    * @param codingStep
    */
-  protected void performCodingStep(ErasureCodingStep codingStep) {
+  protected void performCodingStep(ErasureCodingStep codingStep)
+      throws IOException {
     // Pretend that we're opening these input blocks and output blocks.
     ECBlock[] inputBlocks = codingStep.getInputBlocks();
     ECBlock[] outputBlocks = codingStep.getOutputBlocks();

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHErasureCoderBase.java

@@ -20,6 +20,10 @@ package org.apache.hadoop.io.erasurecode.coder;
 import org.apache.hadoop.io.erasurecode.ECBlock;
 import org.apache.hadoop.io.erasurecode.ECChunk;
 
+import java.io.IOException;
+
+import static org.junit.Assert.fail;
+
 
 /**
  * Erasure coder test base with utilities for hitchhiker.
@@ -53,7 +57,11 @@ public abstract class TestHHErasureCoderBase extends TestErasureCoderBase{
       }
 
       // Given the input chunks and output chunk buffers, just call it !
-      codingStep.performCoding(inputChunks, outputChunks);
+      try {
+        codingStep.performCoding(inputChunks, outputChunks);
+      } catch (IOException e) {
+        fail("Unexpected IOException: " + e.getMessage());
+      }
     }
 
     codingStep.finish();

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java

@@ -22,6 +22,7 @@ import com.google.common.base.Preconditions;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 import org.apache.hadoop.util.StopWatch;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.text.DecimalFormat;
 import java.util.ArrayList;
@@ -232,7 +233,7 @@ public final class RawErasureCoderBenchmark {
     }
   }
 
-  private static RawErasureEncoder getRawEncoder(int index) {
+  private static RawErasureEncoder getRawEncoder(int index) throws IOException {
     RawErasureEncoder encoder =
         CODER_MAKERS.get(index).createEncoder(BenchData.OPTIONS);
     final boolean isDirect = encoder.preferDirectBuffer();
@@ -242,7 +243,7 @@ public final class RawErasureCoderBenchmark {
     return encoder;
   }
 
-  private static RawErasureDecoder getRawDecoder(int index) {
+  private static RawErasureDecoder getRawDecoder(int index) throws IOException {
     RawErasureDecoder decoder =
         CODER_MAKERS.get(index).createDecoder(BenchData.OPTIONS);
     final boolean isDirect = decoder.preferDirectBuffer();
@@ -341,11 +342,11 @@ public final class RawErasureCoderBenchmark {
       System.arraycopy(inputs, 0, decodeInputs, 0, NUM_DATA_UNITS);
     }
 
-    public void encode(RawErasureEncoder encoder) {
+    public void encode(RawErasureEncoder encoder) throws IOException {
       encoder.encode(inputs, outputs);
     }
 
-    public void decode(RawErasureDecoder decoder) {
+    public void decode(RawErasureDecoder decoder) throws IOException {
       decoder.decode(decodeInputs, ERASED_INDEXES, outputs);
     }
   }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestDummyRawCoder.java

@@ -18,9 +18,11 @@
 package org.apache.hadoop.io.erasurecode.rawcoder;
 
 import org.apache.hadoop.io.erasurecode.ECChunk;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 /**
@@ -59,7 +61,11 @@ public class TestDummyRawCoder extends TestRawCoderBase {
     ECChunk[] dataChunks = prepareDataChunksForEncoding();
     markChunks(dataChunks);
     ECChunk[] parityChunks = prepareParityChunksForEncoding();
-    encoder.encode(dataChunks, parityChunks);
+    try {
+      encoder.encode(dataChunks, parityChunks);
+    } catch (IOException e) {
+      Assert.fail("Unexpected IOException: " + e.getMessage());
+    }
     compareAndVerify(parityChunks, getEmptyChunks(parityChunks.length));
 
     // Decode
@@ -69,7 +75,12 @@ public class TestDummyRawCoder extends TestRawCoderBase {
         dataChunks, parityChunks);
     ensureOnlyLeastRequiredChunks(inputChunks);
     ECChunk[] recoveredChunks = prepareOutputChunksForDecoding();
-    decoder.decode(inputChunks, getErasedIndexesForDecoding(), recoveredChunks);
+    try {
+      decoder.decode(inputChunks, getErasedIndexesForDecoding(),
+          recoveredChunks);
+    } catch (IOException e) {
+      Assert.fail("Unexpected IOException: " + e.getMessage());
+    }
     compareAndVerify(recoveredChunks, getEmptyChunks(recoveredChunks.length));
   }
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeRSRawCoder.java

@@ -118,4 +118,10 @@ public class TestNativeRSRawCoder extends TestRSRawCoderBase {
     prepare(null, 10, 4, new int[] {0}, new int[] {0});
     testCodingDoMixAndTwice();
   }
+
+  @Test
+  public void testAfterRelease63() throws Exception {
+    prepare(6, 3, null, null);
+    testAfterRelease();
+  }
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeXORRawCoder.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.io.erasurecode.rawcoder;
 import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
 import org.junit.Assume;
 import org.junit.Before;
+import org.junit.Test;
 
 /**
  * Test NativeXOR encoding and decoding.
@@ -33,4 +34,10 @@ public class TestNativeXORRawCoder extends TestXORRawCoderBase {
     this.decoderFactoryClass = NativeXORRawErasureCoderFactory.class;
     setAllowDump(true);
   }
+
+  @Test
+  public void testAfterRelease63() throws Exception {
+    prepare(6, 3, null, null);
+    testAfterRelease();
+  }
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java

@@ -20,9 +20,12 @@ package org.apache.hadoop.io.erasurecode.rawcoder;
 import org.apache.hadoop.io.erasurecode.ECChunk;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 import org.apache.hadoop.io.erasurecode.TestCoderBase;
+import org.apache.hadoop.test.LambdaTestUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.io.IOException;
+
 /**
  * Raw coder test base with utilities.
  */
@@ -104,6 +107,28 @@ public abstract class TestRawCoderBase extends TestCoderBase {
     }
   }
 
+  /**
+   * Test encode / decode after release(). It should raise IOException.
+   *
+   * @throws Exception
+   */
+  void testAfterRelease() throws Exception {
+    prepareCoders(true);
+    prepareBufferAllocator(true);
+
+    encoder.release();
+    final ECChunk[] data = prepareDataChunksForEncoding();
+    final ECChunk[] parity = prepareParityChunksForEncoding();
+    LambdaTestUtils.intercept(IOException.class, "closed",
+        () -> encoder.encode(data, parity));
+
+    decoder.release();
+    final ECChunk[] in = prepareInputChunksForDecoding(data, parity);
+    final ECChunk[] out = prepareOutputChunksForDecoding();
+    LambdaTestUtils.intercept(IOException.class, "closed",
+        () -> decoder.decode(in, getErasedIndexesForDecoding(), out));
+  }
+
   @Test
   public void testCodingWithErasingTooMany() {
     try {
@@ -121,6 +146,16 @@ public abstract class TestRawCoderBase extends TestCoderBase {
     }
   }
 
+  @Test
+  public void testIdempotentReleases() {
+    prepareCoders(true);
+
+    for (int i = 0; i < 3; i++) {
+      encoder.release();
+      decoder.release();
+    }
+  }
+
   private void performTestCoding(int chunkSize, boolean usingSlicedBuffer,
                                  boolean useBadInput, boolean useBadOutput,
                                  boolean allowChangeInputs) {
@@ -144,7 +179,11 @@ public abstract class TestRawCoderBase extends TestCoderBase {
     ECChunk[] clonedDataChunks = cloneChunksWithData(dataChunks);
     markChunks(dataChunks);
 
-    encoder.encode(dataChunks, parityChunks);
+    try {
+      encoder.encode(dataChunks, parityChunks);
+    } catch (IOException e) {
+      Assert.fail("Should not get IOException: " + e.getMessage());
+    }
     dumpChunks("Encoded parity chunks", parityChunks);
 
     if (!allowChangeInputs) {
@@ -174,7 +213,12 @@ public abstract class TestRawCoderBase extends TestCoderBase {
     }
 
     dumpChunks("Decoding input chunks", inputChunks);
-    decoder.decode(inputChunks, getErasedIndexesForDecoding(), recoveredChunks);
+    try {
+      decoder.decode(inputChunks, getErasedIndexesForDecoding(),
+          recoveredChunks);
+    } catch (IOException e) {
+      Assert.fail("Should not get IOException: " + e.getMessage());
+    }
     dumpChunks("Decoded/recovered chunks", recoveredChunks);
 
     if (!allowChangeInputs) {
@@ -268,7 +312,11 @@ public abstract class TestRawCoderBase extends TestCoderBase {
     ECChunk[] dataChunks = prepareDataChunksForEncoding();
     ECChunk[] parityChunks = prepareParityChunksForEncoding();
     ECChunk[] clonedDataChunks = cloneChunksWithData(dataChunks);
-    encoder.encode(dataChunks, parityChunks);
+    try {
+      encoder.encode(dataChunks, parityChunks);
+    } catch (IOException e) {
+      Assert.fail("Should not get IOException: " + e.getMessage());
+    }
     verifyBufferPositionAtEnd(dataChunks);
 
     // verify decode
@@ -277,7 +325,12 @@ public abstract class TestRawCoderBase extends TestCoderBase {
         clonedDataChunks, parityChunks);
     ensureOnlyLeastRequiredChunks(inputChunks);
     ECChunk[] recoveredChunks = prepareOutputChunksForDecoding();
-    decoder.decode(inputChunks, getErasedIndexesForDecoding(), recoveredChunks);
+    try {
+      decoder.decode(inputChunks, getErasedIndexesForDecoding(),
+          recoveredChunks);
+    } catch (IOException e) {
+      Assert.fail("Should not get IOException: " + e.getMessage());
+    }
     verifyBufferPositionAtEnd(inputChunks);
   }
 

hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java

@@ -326,4 +326,8 @@ public class KMSACLs implements Runnable, KeyACLs {
         || whitelistKeyAcls.containsKey(opType));
   }
 
+  @VisibleForTesting
+  void forceNextReloadForTesting() {
+    lastReload = 0;
+  }
 }

hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java

@@ -20,6 +20,8 @@ package org.apache.hadoop.crypto.key.kms.server;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.net.MalformedURLException;
@@ -31,6 +33,8 @@ import java.net.URL;
 @InterfaceAudience.Private
 public class KMSConfiguration {
 
+  static final Logger LOG = LoggerFactory.getLogger(KMSConfiguration.class);
+
   public static final String KMS_CONFIG_DIR = "kms.config.dir";
   public static final String KMS_DEFAULT_XML = "kms-default.xml";
   public static final String KMS_SITE_XML = "kms-site.xml";
@@ -72,6 +76,15 @@ public class KMSConfiguration {
   public static final String KMS_AUDIT_AGGREGATION_WINDOW = CONFIG_PREFIX +
       "audit.aggregation.window.ms";
 
+  // Process name shown in metrics
+  public static final String METRICS_PROCESS_NAME_KEY =
+      CONFIG_PREFIX + "metrics.process.name";
+  public static final String METRICS_PROCESS_NAME_DEFAULT = "KMS";
+
+  // Session id for metrics
+  public static final String METRICS_SESSION_ID_KEY =
+      CONFIG_PREFIX + "metrics.session.id";
+
   // KMS Audit logger classes to use
   public static final String KMS_AUDIT_LOGGER_KEY = CONFIG_PREFIX +
       "audit.logger";
@@ -138,6 +151,8 @@ public class KMSConfiguration {
             "' must be an absolute path: " + confDir);
       }
       File f = new File(confDir, KMS_ACLS_XML);
+      LOG.trace("Checking file {}, modification time is {}, last reload time is"
+          + " {}", f.getPath(), f.lastModified(), time);
       // at least 100ms newer than time, we do this to ensure the file
       // has been properly closed/flushed
       newer = f.lastModified() - time > 100;

hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java

@@ -27,12 +27,19 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.ConfigurationWithLogging;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.source.JvmMetrics;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.security.ssl.SSLFactory;
+import org.apache.hadoop.util.JvmPauseMonitor;
 import org.apache.hadoop.util.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.crypto.key.kms.server.KMSConfiguration.METRICS_PROCESS_NAME_DEFAULT;
+import static org.apache.hadoop.crypto.key.kms.server.KMSConfiguration.METRICS_PROCESS_NAME_KEY;
+import static org.apache.hadoop.crypto.key.kms.server.KMSConfiguration.METRICS_SESSION_ID_KEY;
+
 /**
  * The KMS web server.
  */
@@ -46,6 +53,9 @@ public class KMSWebServer {
 
   private final HttpServer2 httpServer;
   private final String scheme;
+  private final String processName;
+  private final String sessionId;
+  private final JvmPauseMonitor pauseMonitor;
 
   KMSWebServer(Configuration conf, Configuration sslConf) throws Exception {
     // Override configuration with deprecated environment variables.
@@ -73,6 +83,11 @@ public class KMSWebServer {
     boolean sslEnabled = conf.getBoolean(KMSConfiguration.SSL_ENABLED_KEY,
         KMSConfiguration.SSL_ENABLED_DEFAULT);
     scheme = sslEnabled ? HttpServer2.HTTPS_SCHEME : HttpServer2.HTTP_SCHEME;
+    processName =
+        conf.get(METRICS_PROCESS_NAME_KEY, METRICS_PROCESS_NAME_DEFAULT);
+    sessionId = conf.get(METRICS_SESSION_ID_KEY);
+    pauseMonitor = new JvmPauseMonitor();
+    pauseMonitor.init(conf);
 
     String host = conf.get(KMSConfiguration.HTTP_HOST_KEY,
         KMSConfiguration.HTTP_HOST_DEFAULT);
@@ -113,6 +128,11 @@ public class KMSWebServer {
 
   public void start() throws IOException {
     httpServer.start();
+
+    DefaultMetricsSystem.initialize(processName);
+    final JvmMetrics jm = JvmMetrics.initSingleton(processName, sessionId);
+    jm.setPauseMonitor(pauseMonitor);
+    pauseMonitor.start();
   }
 
   public boolean isRunning() {
@@ -125,6 +145,10 @@ public class KMSWebServer {
 
   public void stop() throws Exception {
     httpServer.stop();
+
+    pauseMonitor.stop();
+    JvmMetrics.shutdownSingleton();
+    DefaultMetricsSystem.shutdown();
   }
 
   public URL getKMSUrl() {

hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java

@@ -33,6 +33,7 @@ import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
 import org.apache.hadoop.crypto.key.kms.KMSDelegationToken;
 import org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider;
 import org.apache.hadoop.crypto.key.kms.ValueQueue;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.minikdc.MiniKdc;
 import org.apache.hadoop.security.Credentials;
@@ -54,6 +55,7 @@ import org.junit.Test;
 import org.junit.rules.Timeout;
 import org.mockito.Mockito;
 import org.mockito.internal.util.reflection.Whitebox;
+import org.slf4j.event.Level;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -97,6 +99,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.Mockito.when;
 
@@ -1637,13 +1640,12 @@ public class TestKMS {
         //stop the reloader, to avoid running while we are writing the new file
         KMSWebApp.getACLs().stopReloader();
 
+        GenericTestUtils.setLogLevel(KMSConfiguration.LOG, Level.TRACE);
         // test ACL reloading
-        Thread.sleep(10); // to ensure the ACLs file modifiedTime is newer
         conf.set(KMSACLs.Type.CREATE.getAclConfigKey(), "foo");
         conf.set(KMSACLs.Type.GENERATE_EEK.getAclConfigKey(), "foo");
         writeConf(testDir, conf);
-        Thread.sleep(1000);
-
+        KMSWebApp.getACLs().forceNextReloadForTesting();
         KMSWebApp.getACLs().run(); // forcing a reload by hand.
 
         // should not be able to create a key now
@@ -1882,7 +1884,7 @@ public class TestKMS {
   public void testKMSTimeout() throws Exception {
     File confDir = getTestDir();
     Configuration conf = createBaseKMSConf(confDir);
-    conf.setInt(KMSClientProvider.TIMEOUT_ATTR, 1);
+    conf.setInt(CommonConfigurationKeysPublic.KMS_CLIENT_TIMEOUT_SECONDS, 1);
     writeConf(confDir, conf);
 
     ServerSocket sock;
@@ -2693,4 +2695,38 @@ public class TestKMS {
     });
   }
 
+  /*
+   * Test the jmx page can return, and contains the basic JvmMetrics. Only
+   * testing in simple mode since the page content is the same, kerberized
+   * or not.
+   */
+  @Test
+  public void testKMSJMX() throws Exception {
+    Configuration conf = new Configuration();
+    final File confDir = getTestDir();
+    conf = createBaseKMSConf(confDir, conf);
+    final String processName = "testkmsjmx";
+    conf.set(KMSConfiguration.METRICS_PROCESS_NAME_KEY, processName);
+    writeConf(confDir, conf);
+
+    runServer(null, null, confDir, new KMSCallable<Void>() {
+      @Override
+      public Void call() throws Exception {
+        final URL jmxUrl = new URL(
+            getKMSUrl() + "/jmx?user.name=whatever&qry=Hadoop:service="
+                + processName + ",name=JvmMetrics");
+        LOG.info("Requesting jmx from " + jmxUrl);
+        final StringBuilder sb = new StringBuilder();
+        final InputStream in = jmxUrl.openConnection().getInputStream();
+        final byte[] buffer = new byte[64 * 1024];
+        int len;
+        while ((len = in.read(buffer)) > 0) {
+          sb.append(new String(buffer, 0, len));
+        }
+        LOG.info("jmx returned: " + sb.toString());
+        assertTrue(sb.toString().contains("JvmMetrics"));
+        return null;
+      }
+    });
+  }
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java

@@ -82,6 +82,12 @@ public class DFSStripedOutputStream extends DFSOutputStream
     implements StreamCapabilities {
   private static final ByteBufferPool BUFFER_POOL = new ElasticByteBufferPool();
 
+  /**
+   * OutputStream level last exception, will be used to indicate the fatal
+   * exception of this stream, i.e., being aborted.
+   */
+  private final ExceptionLastSeen exceptionLastSeen = new ExceptionLastSeen();
+
   static class MultipleBlockingQueue<T> {
     private final List<BlockingQueue<T>> queues;
 
@@ -198,7 +204,7 @@ public class DFSStripedOutputStream extends DFSOutputStream
     private final ByteBuffer[] buffers;
     private final byte[][] checksumArrays;
 
-    CellBuffers(int numParityBlocks) throws InterruptedException{
+    CellBuffers(int numParityBlocks) {
       if (cellSize % bytesPerChecksum != 0) {
         throw new HadoopIllegalArgumentException("Invalid values: "
             + HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY + " (="
@@ -304,12 +310,7 @@ public class DFSStripedOutputStream extends DFSOutputStream
         ecPolicy.getCodecName(), coderOptions);
 
     coordinator = new Coordinator(numAllBlocks);
-    try {
-      cellBuffers = new CellBuffers(numParityBlocks);
-    } catch (InterruptedException ie) {
-      throw DFSUtilClient.toInterruptedIOException(
-          "Failed to create cell buffers", ie);
-    }
+    cellBuffers = new CellBuffers(numParityBlocks);
 
     streamers = new ArrayList<>(numAllBlocks);
     for (short i = 0; i < numAllBlocks; i++) {
@@ -360,7 +361,7 @@ public class DFSStripedOutputStream extends DFSOutputStream
    * @param buffers data buffers + parity buffers
    */
   private static void encode(RawErasureEncoder encoder, int numData,
-      ByteBuffer[] buffers) {
+      ByteBuffer[] buffers) throws IOException {
     final ByteBuffer[] dataBuffers = new ByteBuffer[numData];
     final ByteBuffer[] parityBuffers = new ByteBuffer[buffers.length - numData];
     System.arraycopy(buffers, 0, dataBuffers, 0, dataBuffers.length);
@@ -976,12 +977,9 @@ public class DFSStripedOutputStream extends DFSOutputStream
       if (isClosed()) {
         return;
       }
-      for (StripedDataStreamer streamer : streamers) {
-        streamer.getLastException().set(
-            new IOException("Lease timeout of "
-                + (dfsClient.getConf().getHdfsTimeout() / 1000)
-                + " seconds expired."));
-      }
+      exceptionLastSeen.set(new IOException("Lease timeout of "
+          + (dfsClient.getConf().getHdfsTimeout() / 1000)
+          + " seconds expired."));
 
       try {
         closeThreads(true);
@@ -1138,18 +1136,26 @@ public class DFSStripedOutputStream extends DFSOutputStream
   @Override
   protected synchronized void closeImpl() throws IOException {
     if (isClosed()) {
+      exceptionLastSeen.check(true);
+
+      // Writing to at least {dataUnits} replicas can be considered as success,
+      // and the rest of data can be recovered.
+      final int minReplication = ecPolicy.getNumDataUnits();
+      int goodStreamers = 0;
       final MultipleIOException.Builder b = new MultipleIOException.Builder();
-      for(int i = 0; i < streamers.size(); i++) {
-        final StripedDataStreamer si = getStripedDataStreamer(i);
+      for (final StripedDataStreamer si : streamers) {
         try {
           si.getLastException().check(true);
+          goodStreamers++;
         } catch (IOException e) {
           b.add(e);
         }
       }
-      final IOException ioe = b.build();
-      if (ioe != null) {
-        throw ioe;
+      if (goodStreamers < minReplication) {
+        final IOException ioe = b.build();
+        if (ioe != null) {
+          throw ioe;
+        }
       }
       return;
     }
@@ -1188,9 +1194,10 @@ public class DFSStripedOutputStream extends DFSOutputStream
         }
       } finally {
         // Failures may happen when flushing data/parity data out. Exceptions
-        // may be thrown if more than 3 streamers fail, or updatePipeline RPC
-        // fails. Streamers may keep waiting for the new block/GS information.
-        // Thus need to force closing these threads.
+        // may be thrown if the number of failed streamers is more than the
+        // number of parity blocks, or updatePipeline RPC fails. Streamers may
+        // keep waiting for the new block/GS information. Thus need to force
+        // closing these threads.
         closeThreads(true);
       }
 

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java

@@ -285,39 +285,22 @@ class DataStreamer extends Daemon {
     packets.clear();
   }
 
-  class LastExceptionInStreamer {
-    private IOException thrown;
-
-    synchronized void set(Throwable t) {
-      assert t != null;
-      this.thrown = t instanceof IOException ?
-          (IOException) t : new IOException(t);
-    }
-
-    synchronized void clear() {
-      thrown = null;
-    }
-
-    /** Check if there already is an exception. */
+  class LastExceptionInStreamer extends ExceptionLastSeen {
+    /**
+     * Check if there already is an exception.
+     */
+    @Override
     synchronized void check(boolean resetToNull) throws IOException {
+      final IOException thrown = get();
       if (thrown != null) {
         if (LOG.isTraceEnabled()) {
           // wrap and print the exception to know when the check is called
           LOG.trace("Got Exception while checking, " + DataStreamer.this,
               new Throwable(thrown));
         }
-        final IOException e = thrown;
-        if (resetToNull) {
-          thrown = null;
-        }
-        throw e;
+        super.check(resetToNull);
       }
     }
-
-    synchronized void throwException4Close() throws IOException {
-      check(false);
-      throw new ClosedChannelException();
-    }
   }
 
   enum ErrorType {

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java

@@ -2783,8 +2783,7 @@ public class DistributedFileSystem extends FileSystem {
             }
           }
         } else {
-          Path userTrash = new Path(ezTrashRoot, System.getProperty(
-              "user.name"));
+          Path userTrash = new Path(ezTrashRoot, dfs.ugi.getShortUserName());
           try {
             ret.add(getFileStatus(userTrash));
           } catch (FileNotFoundException ignored) {

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ExceptionLastSeen.java

@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import java.io.IOException;
+import java.nio.channels.ClosedChannelException;
+
+/**
+ * The exception last seen by the {@link DataStreamer} or
+ * {@link DFSOutputStream}.
+ */
+@InterfaceAudience.Private
+class ExceptionLastSeen {
+  private IOException thrown;
+
+  /** Get the last seen exception. */
+  synchronized protected IOException get() {
+    return thrown;
+  }
+
+  /**
+   * Set the last seen exception.
+   * @param t the exception.
+   */
+  synchronized void set(Throwable t) {
+    assert t != null;
+    this.thrown = t instanceof IOException ?
+        (IOException) t : new IOException(t);
+  }
+
+  /** Clear the last seen exception. */
+  synchronized void clear() {
+    thrown = null;
+  }
+
+  /**
+   * Check if there already is an exception. Throw the exception if exist.
+   *
+   * @param resetToNull set to true to reset exception to null after calling
+   *                    this function.
+   * @throws IOException on existing IOException.
+   */
+  synchronized void check(boolean resetToNull) throws IOException {
+    if (thrown != null) {
+      final IOException e = thrown;
+      if (resetToNull) {
+        thrown = null;
+      }
+      throw e;
+    }
+  }
+
+  synchronized void throwException4Close() throws IOException {
+    check(false);
+    throw new ClosedChannelException();
+  }
+}

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java

@@ -27,6 +27,7 @@ import org.apache.hadoop.io.erasurecode.ECChunk;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
 import org.apache.hadoop.hdfs.DFSUtilClient.CorruptedBlocks;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 /**
@@ -68,7 +69,7 @@ class PositionStripeReader extends StripeReader {
   }
 
   @Override
-  void decode() {
+  void decode() throws IOException {
     finalizeDecodeInputs();
     decodeAndFillBuffer(true);
   }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java

@@ -27,6 +27,7 @@ import org.apache.hadoop.io.erasurecode.ECChunk;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
 import org.apache.hadoop.hdfs.DFSUtilClient.CorruptedBlocks;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 /**
@@ -88,7 +89,7 @@ class StatefulStripeReader extends StripeReader {
   }
 
   @Override
-  void decode() {
+  void decode() throws IOException {
     finalizeDecodeInputs();
     decodeAndFillBuffer(false);
   }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java

@@ -149,10 +149,11 @@ abstract class StripeReader {
    */
   abstract boolean prepareParityChunk(int index);
 
-  /*
+  /**
    * Decode to get the missing data.
+   * @throws IOException if the decoder is closed.
    */
-  abstract void decode();
+  abstract void decode() throws IOException;
 
   /*
    * Default close do nothing.
@@ -408,7 +409,7 @@ abstract class StripeReader {
   /**
    * Decode based on the given input buffers and erasure coding policy.
    */
-  void decodeAndFillBuffer(boolean fillBuffer) {
+  void decodeAndFillBuffer(boolean fillBuffer) throws IOException {
     // Step 1: prepare indices and output buffers for missing data units
     int[] decodeIndices = prepareErasedIndices();
 

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java

@@ -1495,6 +1495,15 @@ public class WebHdfsFileSystem extends FileSystem
     }
   }
 
+  /**
+   * Get {@link FileStatus} of files/directories in the given path. If path
+   * corresponds to a file then {@link FileStatus} of that file is returned.
+   * Else if path represents a directory then {@link FileStatus} of all
+   * files/directories inside given path is returned.
+   *
+   * @param f given path
+   * @return the statuses of the files/directories in the given path
+   */
   @Override
   public FileStatus[] listStatus(final Path f) throws IOException {
     statistics.incrementReadOps(1);
@@ -1519,6 +1528,13 @@ public class WebHdfsFileSystem extends FileSystem
 
   private static final byte[] EMPTY_ARRAY = new byte[] {};
 
+  /**
+   * Get DirectoryEntries of the given path. DirectoryEntries contains an array
+   * of {@link FileStatus}, as well as iteration information.
+   *
+   * @param f given path
+   * @return DirectoryEntries for given path
+   */
   @Override
   public DirectoryEntries listStatusBatch(Path f, byte[] token) throws
       FileNotFoundException, IOException {

hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java

@@ -705,14 +705,13 @@ public class HttpFSFileSystem extends FileSystem
   }
 
   /**
-   * List the statuses of the files/directories in the given path if the path is
-   * a directory.
+   * Get {@link FileStatus} of files/directories in the given path. If path
+   * corresponds to a file then {@link FileStatus} of that file is returned.
+   * Else if path represents a directory then {@link FileStatus} of all
+   * files/directories inside given path is returned.
    *
    * @param f given path
-   *
-   * @return the statuses of the files/directories in the given patch
-   *
-   * @throws IOException
+   * @return the statuses of the files/directories in the given path
    */
   @Override
   public FileStatus[] listStatus(Path f) throws IOException {
@@ -725,6 +724,13 @@ public class HttpFSFileSystem extends FileSystem
     return toFileStatuses(json, f);
   }
 
+  /**
+   * Get {@link DirectoryEntries} of the given path. {@link DirectoryEntries}
+   * contains an array of {@link FileStatus}, as well as iteration information.
+   *
+   * @param f given path
+   * @return {@link DirectoryEntries} for given path
+   */
   @Override
   public DirectoryEntries listStatusBatch(Path f, byte[] token) throws
       FileNotFoundException, IOException {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java

@@ -326,7 +326,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
 
   public static final String  DFS_NAMENODE_EDITS_ASYNC_LOGGING =
       "dfs.namenode.edits.asynclogging";
-  public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = false;
+  public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = true;
 
   public static final String  DFS_LIST_LIMIT = "dfs.ls.limit";
   public static final int     DFS_LIST_LIMIT_DEFAULT = 1000;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java

@@ -49,7 +49,7 @@ interface AsyncLogger {
   
   interface Factory {
     AsyncLogger createLogger(Configuration conf, NamespaceInfo nsInfo,
-        String journalId, InetSocketAddress addr);
+        String journalId, String nameServiceId, InetSocketAddress addr);
   }
 
   /**

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java

@@ -95,6 +95,8 @@ public class IPCLoggerChannel implements AsyncLogger {
   private long committedTxId = HdfsServerConstants.INVALID_TXID;
   
   private final String journalId;
+  private final String nameServiceId;
+
   private final NamespaceInfo nsInfo;
 
   private URL httpServerURL;
@@ -152,8 +154,8 @@ public class IPCLoggerChannel implements AsyncLogger {
   static final Factory FACTORY = new AsyncLogger.Factory() {
     @Override
     public AsyncLogger createLogger(Configuration conf, NamespaceInfo nsInfo,
-        String journalId, InetSocketAddress addr) {
-      return new IPCLoggerChannel(conf, nsInfo, journalId, addr);
+        String journalId, String nameServiceId, InetSocketAddress addr) {
+      return new IPCLoggerChannel(conf, nsInfo, journalId, nameServiceId, addr);
     }
   };
 
@@ -161,11 +163,19 @@ public class IPCLoggerChannel implements AsyncLogger {
       NamespaceInfo nsInfo,
       String journalId,
       InetSocketAddress addr) {
+    this(conf, nsInfo, journalId, null, addr);
+  }
+
+  public IPCLoggerChannel(Configuration conf,
+                          NamespaceInfo nsInfo,
+                          String journalId,
+                          String nameServiceId,
+                          InetSocketAddress addr) {
     this.conf = conf;
     this.nsInfo = nsInfo;
     this.journalId = journalId;
+    this.nameServiceId = nameServiceId;
     this.addr = addr;
-    
     this.queueSizeLimitBytes = 1024 * 1024 * conf.getInt(
         DFSConfigKeys.DFS_QJOURNAL_QUEUE_SIZE_LIMIT_KEY,
         DFSConfigKeys.DFS_QJOURNAL_QUEUE_SIZE_LIMIT_DEFAULT);
@@ -286,7 +296,8 @@ public class IPCLoggerChannel implements AsyncLogger {
 
   private synchronized RequestInfo createReqInfo() {
     Preconditions.checkState(epoch > 0, "bad epoch: " + epoch);
-    return new RequestInfo(journalId, epoch, ipcSerial++, committedTxId);
+    return new RequestInfo(journalId, nameServiceId,
+        epoch, ipcSerial++, committedTxId);
   }
 
   @VisibleForTesting
@@ -330,7 +341,7 @@ public class IPCLoggerChannel implements AsyncLogger {
     return singleThreadExecutor.submit(new Callable<Boolean>() {
       @Override
       public Boolean call() throws IOException {
-        return getProxy().isFormatted(journalId);
+        return getProxy().isFormatted(journalId, nameServiceId);
       }
     });
   }
@@ -341,7 +352,7 @@ public class IPCLoggerChannel implements AsyncLogger {
       @Override
       public GetJournalStateResponseProto call() throws IOException {
         GetJournalStateResponseProto ret =
-            getProxy().getJournalState(journalId);
+            getProxy().getJournalState(journalId, nameServiceId);
         constructHttpServerURI(ret);
         return ret;
       }
@@ -354,7 +365,7 @@ public class IPCLoggerChannel implements AsyncLogger {
     return singleThreadExecutor.submit(new Callable<NewEpochResponseProto>() {
       @Override
       public NewEpochResponseProto call() throws IOException {
-        return getProxy().newEpoch(journalId, nsInfo, epoch);
+        return getProxy().newEpoch(journalId, nameServiceId, nsInfo, epoch);
       }
     });
   }
@@ -495,7 +506,7 @@ public class IPCLoggerChannel implements AsyncLogger {
     return singleThreadExecutor.submit(new Callable<Void>() {
       @Override
       public Void call() throws Exception {
-        getProxy().format(journalId, nsInfo);
+        getProxy().format(journalId, nameServiceId, nsInfo);
         return null;
       }
     });
@@ -554,7 +565,7 @@ public class IPCLoggerChannel implements AsyncLogger {
       @Override
       public RemoteEditLogManifest call() throws IOException {
         GetEditLogManifestResponseProto ret = getProxy().getEditLogManifest(
-            journalId, fromTxnId, inProgressOk);
+            journalId, nameServiceId, fromTxnId, inProgressOk);
         // Update the http port, since we need this to build URLs to any of the
         // returned logs.
         constructHttpServerURI(ret);
@@ -573,7 +584,7 @@ public class IPCLoggerChannel implements AsyncLogger {
           // force an RPC call so we know what the HTTP port should be if it
           // haven't done so.
           GetJournalStateResponseProto ret = getProxy().getJournalState(
-              journalId);
+              journalId, nameServiceId);
           constructHttpServerURI(ret);
         }
         return getProxy().prepareRecovery(createReqInfo(), segmentTxId);
@@ -620,7 +631,7 @@ public class IPCLoggerChannel implements AsyncLogger {
     return singleThreadExecutor.submit(new Callable<Void>() {
       @Override
       public Void call() throws IOException {
-        getProxy().doFinalize(journalId);
+        getProxy().doFinalize(journalId, nameServiceId);
         return null;
       }
     });
@@ -632,8 +643,8 @@ public class IPCLoggerChannel implements AsyncLogger {
     return singleThreadExecutor.submit(new Callable<Boolean>() {
       @Override
       public Boolean call() throws IOException {
-        return getProxy().canRollBack(journalId, storage, prevStorage,
-            targetLayoutVersion);
+        return getProxy().canRollBack(journalId, nameServiceId,
+            storage, prevStorage, targetLayoutVersion);
       }
     });
   }
@@ -643,7 +654,7 @@ public class IPCLoggerChannel implements AsyncLogger {
     return singleThreadExecutor.submit(new Callable<Void>() {
       @Override
       public Void call() throws IOException {
-        getProxy().doRollback(journalId);
+        getProxy().doRollback(journalId, nameServiceId);
         return null;
       }
     });
@@ -654,7 +665,7 @@ public class IPCLoggerChannel implements AsyncLogger {
     return singleThreadExecutor.submit(new Callable<Void>() {
       @Override
       public Void call() throws IOException {
-        getProxy().discardSegments(journalId, startTxId);
+        getProxy().discardSegments(journalId, nameServiceId, startTxId);
         return null;
       }
     });
@@ -665,7 +676,7 @@ public class IPCLoggerChannel implements AsyncLogger {
     return singleThreadExecutor.submit(new Callable<Long>() {
       @Override
       public Long call() throws IOException {
-        return getProxy().getJournalCTime(journalId);
+        return getProxy().getJournalCTime(journalId, nameServiceId);
       }
     });
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java

@@ -85,26 +85,44 @@ public class QuorumJournalManager implements JournalManager {
   private final Configuration conf;
   private final URI uri;
   private final NamespaceInfo nsInfo;
+  private final String nameServiceId;
   private boolean isActiveWriter;
   
   private final AsyncLoggerSet loggers;
 
   private int outputBufferCapacity = 512 * 1024;
   private final URLConnectionFactory connectionFactory;
+
+  @VisibleForTesting
+  public QuorumJournalManager(Configuration conf,
+                              URI uri,
+                              NamespaceInfo nsInfo) throws IOException {
+    this(conf, uri, nsInfo, null, IPCLoggerChannel.FACTORY);
+  }
   
   public QuorumJournalManager(Configuration conf,
-      URI uri, NamespaceInfo nsInfo) throws IOException {
-    this(conf, uri, nsInfo, IPCLoggerChannel.FACTORY);
+      URI uri, NamespaceInfo nsInfo, String nameServiceId) throws IOException {
+    this(conf, uri, nsInfo, nameServiceId, IPCLoggerChannel.FACTORY);
   }
+
+  @VisibleForTesting
+  QuorumJournalManager(Configuration conf,
+                       URI uri, NamespaceInfo nsInfo,
+                       AsyncLogger.Factory loggerFactory) throws IOException {
+    this(conf, uri, nsInfo, null, loggerFactory);
+
+  }
+
   
   QuorumJournalManager(Configuration conf,
-      URI uri, NamespaceInfo nsInfo,
+      URI uri, NamespaceInfo nsInfo, String nameServiceId,
       AsyncLogger.Factory loggerFactory) throws IOException {
     Preconditions.checkArgument(conf != null, "must be configured");
 
     this.conf = conf;
     this.uri = uri;
     this.nsInfo = nsInfo;
+    this.nameServiceId = nameServiceId;
     this.loggers = new AsyncLoggerSet(createLoggers(loggerFactory));
     this.connectionFactory = URLConnectionFactory
         .newDefaultURLConnectionFactory(conf);
@@ -142,7 +160,7 @@ public class QuorumJournalManager implements JournalManager {
   
   protected List<AsyncLogger> createLoggers(
       AsyncLogger.Factory factory) throws IOException {
-    return createLoggers(conf, uri, nsInfo, factory);
+    return createLoggers(conf, uri, nsInfo, factory, nameServiceId);
   }
 
   static String parseJournalId(URI uri) {
@@ -354,8 +372,11 @@ public class QuorumJournalManager implements JournalManager {
   }
   
   static List<AsyncLogger> createLoggers(Configuration conf,
-      URI uri, NamespaceInfo nsInfo, AsyncLogger.Factory factory)
-          throws IOException {
+                                         URI uri,
+                                         NamespaceInfo nsInfo,
+                                         AsyncLogger.Factory factory,
+                                         String nameServiceId)
+      throws IOException {
     List<AsyncLogger> ret = Lists.newArrayList();
     List<InetSocketAddress> addrs = Util.getAddressesList(uri);
     if (addrs.size() % 2 == 0) {
@@ -364,7 +385,7 @@ public class QuorumJournalManager implements JournalManager {
     }
     String jid = parseJournalId(uri);
     for (InetSocketAddress addr : addrs) {
-      ret.add(factory.createLogger(conf, nsInfo, jid, addr));
+      ret.add(factory.createLogger(conf, nsInfo, jid, nameServiceId, addr));
     }
     return ret;
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java

@@ -53,26 +53,30 @@ public interface QJournalProtocol {
    * @return true if the given journal has been formatted and
    * contains valid data.
    */
-  public boolean isFormatted(String journalId) throws IOException;
+  boolean isFormatted(String journalId,
+                      String nameServiceId) throws IOException;
 
   /**
    * Get the current state of the journal, including the most recent
    * epoch number and the HTTP port.
    */
-  public GetJournalStateResponseProto getJournalState(String journalId)
+  GetJournalStateResponseProto getJournalState(String journalId,
+                                               String nameServiceId)
       throws IOException;
   
   /**
    * Format the underlying storage for the given namespace.
    */
-  public void format(String journalId,
+  void format(String journalId, String nameServiceId,
       NamespaceInfo nsInfo) throws IOException;
 
   /**
    * Begin a new epoch. See the HDFS-3077 design doc for details.
    */
-  public NewEpochResponseProto newEpoch(String journalId,
-      NamespaceInfo nsInfo, long epoch) throws IOException;
+  NewEpochResponseProto newEpoch(String journalId,
+                                        String nameServiceId,
+                                        NamespaceInfo nsInfo,
+                                        long epoch) throws IOException;
   
   /**
    * Journal edit records.
@@ -130,8 +134,10 @@ public interface QJournalProtocol {
    *        segment       
    * @return a list of edit log segments since the given transaction ID.
    */
-  public GetEditLogManifestResponseProto getEditLogManifest(String jid,
-      long sinceTxId, boolean inProgressOk)
+  GetEditLogManifestResponseProto getEditLogManifest(String jid,
+                                                     String nameServiceId,
+                                                     long sinceTxId,
+                                                     boolean inProgressOk)
       throws IOException;
   
   /**
@@ -147,24 +153,30 @@ public interface QJournalProtocol {
   public void acceptRecovery(RequestInfo reqInfo,
       SegmentStateProto stateToAccept, URL fromUrl) throws IOException;
 
-  public void doPreUpgrade(String journalId) throws IOException;
+  void doPreUpgrade(String journalId) throws IOException;
 
   public void doUpgrade(String journalId, StorageInfo sInfo) throws IOException;
 
-  public void doFinalize(String journalId) throws IOException;
+  void doFinalize(String journalId,
+                         String nameServiceid) throws IOException;
 
-  public Boolean canRollBack(String journalId, StorageInfo storage,
-      StorageInfo prevStorage, int targetLayoutVersion) throws IOException;
+  Boolean canRollBack(String journalId, String nameServiceid,
+                      StorageInfo storage, StorageInfo prevStorage,
+                      int targetLayoutVersion) throws IOException;
 
-  public void doRollback(String journalId) throws IOException;
+  void doRollback(String journalId,
+                         String nameServiceid) throws IOException;
 
   /**
    * Discard journal segments whose first TxId is greater than or equal to the
    * given txid.
    */
   @Idempotent
-  public void discardSegments(String journalId, long startTxId)
+  void discardSegments(String journalId,
+                       String nameServiceId,
+                       long startTxId)
       throws IOException;
 
-  public Long getJournalCTime(String journalId) throws IOException;
+  Long getJournalCTime(String journalId,
+                       String nameServiceId) throws IOException;
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/RequestInfo.java

@@ -26,15 +26,22 @@ public class RequestInfo {
   private long epoch;
   private long ipcSerialNumber;
   private final long committedTxId;
+  private final String nameServiceId;
   
-  public RequestInfo(String jid, long epoch, long ipcSerialNumber,
-      long committedTxId) {
+  public RequestInfo(String jid, String nameServiceId,
+                     long epoch, long ipcSerialNumber,
+                     long committedTxId) {
     this.jid = jid;
+    this.nameServiceId = nameServiceId;
     this.epoch = epoch;
     this.ipcSerialNumber = ipcSerialNumber;
     this.committedTxId = committedTxId;
   }
 
+  public String getNameServiceId() {
+    return nameServiceId;
+  }
+
   public long getEpoch() {
     return epoch;
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java

@@ -101,7 +101,8 @@ public class QJournalProtocolServerSideTranslatorPB implements QJournalProtocolP
       IsFormattedRequestProto request) throws ServiceException {
     try {
       boolean ret = impl.isFormatted(
-          convert(request.getJid()));
+          convert(request.getJid()),
+          request.hasNameServiceId() ? request.getNameServiceId() : null);
       return IsFormattedResponseProto.newBuilder()
           .setIsFormatted(ret)
           .build();
@@ -116,7 +117,8 @@ public class QJournalProtocolServerSideTranslatorPB implements QJournalProtocolP
       GetJournalStateRequestProto request) throws ServiceException {
     try {
       return impl.getJournalState(
-          convert(request.getJid()));
+          convert(request.getJid()),
+          request.hasNameServiceId() ? request.getNameServiceId() : null);
     } catch (IOException ioe) {
       throw new ServiceException(ioe);
     }
@@ -132,6 +134,7 @@ public class QJournalProtocolServerSideTranslatorPB implements QJournalProtocolP
     try {
       return impl.newEpoch(
           request.getJid().getIdentifier(),
+          request.hasNameServiceId() ? request.getNameServiceId() : null,
           PBHelper.convert(request.getNsInfo()),
           request.getEpoch());
     } catch (IOException ioe) {
@@ -143,6 +146,7 @@ public class QJournalProtocolServerSideTranslatorPB implements QJournalProtocolP
       FormatRequestProto request) throws ServiceException {
     try {
       impl.format(request.getJid().getIdentifier(),
+          request.hasNameServiceId() ? request.getNameServiceId() : null,
           PBHelper.convert(request.getNsInfo()));
       return FormatResponseProto.getDefaultInstance();
     } catch (IOException ioe) {
@@ -223,6 +227,7 @@ public class QJournalProtocolServerSideTranslatorPB implements QJournalProtocolP
     try {
       return impl.getEditLogManifest(
           request.getJid().getIdentifier(),
+          request.hasNameServiceId() ? request.getNameServiceId() : null,
           request.getSinceTxId(),
           request.getInProgressOk());
     } catch (IOException e) {
@@ -260,6 +265,8 @@ public class QJournalProtocolServerSideTranslatorPB implements QJournalProtocolP
       QJournalProtocolProtos.RequestInfoProto reqInfo) {
     return new RequestInfo(
         reqInfo.getJournalId().getIdentifier(),
+        reqInfo.hasNameServiceId() ?
+            reqInfo.getNameServiceId() : null,
         reqInfo.getEpoch(),
         reqInfo.getIpcSerialNumber(),
         reqInfo.hasCommittedTxId() ?
@@ -294,7 +301,8 @@ public class QJournalProtocolServerSideTranslatorPB implements QJournalProtocolP
   public DoFinalizeResponseProto doFinalize(RpcController controller,
       DoFinalizeRequestProto request) throws ServiceException {
     try {
-      impl.doFinalize(convert(request.getJid()));
+      impl.doFinalize(convert(request.getJid()),
+          request.hasNameServiceId() ? request.getNameServiceId() : null);
       return DoFinalizeResponseProto.getDefaultInstance();
     } catch (IOException e) {
       throw new ServiceException(e);
@@ -306,7 +314,9 @@ public class QJournalProtocolServerSideTranslatorPB implements QJournalProtocolP
       CanRollBackRequestProto request) throws ServiceException {
     try {
       StorageInfo si = PBHelper.convert(request.getStorage(), NodeType.JOURNAL_NODE);
-      Boolean result = impl.canRollBack(convert(request.getJid()), si,
+      Boolean result = impl.canRollBack(convert(request.getJid()),
+          request.hasNameServiceId() ? request.getNameServiceId() : null,
+          si,
           PBHelper.convert(request.getPrevStorage(), NodeType.JOURNAL_NODE),
           request.getTargetLayoutVersion());
       return CanRollBackResponseProto.newBuilder()
@@ -321,7 +331,7 @@ public class QJournalProtocolServerSideTranslatorPB implements QJournalProtocolP
   public DoRollbackResponseProto doRollback(RpcController controller, DoRollbackRequestProto request)
       throws ServiceException {
     try {
-      impl.doRollback(convert(request.getJid()));
+      impl.doRollback(convert(request.getJid()), request.getNameserviceId());
       return DoRollbackResponseProto.getDefaultInstance();
     } catch (IOException e) {
       throw new ServiceException(e);
@@ -333,7 +343,9 @@ public class QJournalProtocolServerSideTranslatorPB implements QJournalProtocolP
       RpcController controller, DiscardSegmentsRequestProto request)
       throws ServiceException {
     try {
-      impl.discardSegments(convert(request.getJid()), request.getStartTxId());
+      impl.discardSegments(convert(request.getJid()),
+          request.hasNameServiceId() ? request.getNameServiceId() : null,
+          request.getStartTxId());
       return DiscardSegmentsResponseProto.getDefaultInstance();
     } catch (IOException e) {
       throw new ServiceException(e);
@@ -344,7 +356,8 @@ public class QJournalProtocolServerSideTranslatorPB implements QJournalProtocolP
   public GetJournalCTimeResponseProto getJournalCTime(RpcController controller,
       GetJournalCTimeRequestProto request) throws ServiceException {
     try {
-      Long resultCTime = impl.getJournalCTime(convert(request.getJid()));
+      Long resultCTime = impl.getJournalCTime(convert(request.getJid()),
+          request.getNameServiceId());
       return GetJournalCTimeResponseProto.newBuilder()
           .setResultCTime(resultCTime)
           .build();

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java

@@ -93,13 +93,17 @@ public class QJournalProtocolTranslatorPB implements ProtocolMetaInterface,
 
 
   @Override
-  public boolean isFormatted(String journalId) throws IOException {
+  public boolean isFormatted(String journalId,
+                             String nameServiceId) throws IOException {
     try {
-      IsFormattedRequestProto req = IsFormattedRequestProto.newBuilder()
-          .setJid(convertJournalId(journalId))
-          .build();
+      IsFormattedRequestProto.Builder req = IsFormattedRequestProto.newBuilder()
+          .setJid(convertJournalId(journalId));
+      if (nameServiceId != null) {
+        req.setNameServiceId(nameServiceId);
+      }
+
       IsFormattedResponseProto resp = rpcProxy.isFormatted(
-          NULL_CONTROLLER, req);
+          NULL_CONTROLLER, req.build());
       return resp.getIsFormatted();
     } catch (ServiceException e) {
       throw ProtobufHelper.getRemoteException(e);
@@ -107,13 +111,17 @@ public class QJournalProtocolTranslatorPB implements ProtocolMetaInterface,
   }
 
   @Override
-  public GetJournalStateResponseProto getJournalState(String jid)
+  public GetJournalStateResponseProto getJournalState(String jid,
+                                                      String nameServiceId)
       throws IOException {
     try {
-      GetJournalStateRequestProto req = GetJournalStateRequestProto.newBuilder()
-          .setJid(convertJournalId(jid))
-          .build();
-      return rpcProxy.getJournalState(NULL_CONTROLLER, req);
+      GetJournalStateRequestProto.Builder req = GetJournalStateRequestProto
+          .newBuilder()
+          .setJid(convertJournalId(jid));
+      if (nameServiceId != null) {
+        req.setNameServiceId(nameServiceId);
+      }
+      return rpcProxy.getJournalState(NULL_CONTROLLER, req.build());
     } catch (ServiceException e) {
       throw ProtobufHelper.getRemoteException(e);
     }
@@ -126,28 +134,39 @@ public class QJournalProtocolTranslatorPB implements ProtocolMetaInterface,
   }
   
   @Override
-  public void format(String jid, NamespaceInfo nsInfo) throws IOException {
+  public void format(String jid,
+                     String nameServiceId,
+                     NamespaceInfo nsInfo) throws IOException {
     try {
-      FormatRequestProto req = FormatRequestProto.newBuilder()
+      FormatRequestProto.Builder req = FormatRequestProto.newBuilder()
           .setJid(convertJournalId(jid))
-          .setNsInfo(PBHelper.convert(nsInfo))
-          .build();
-      rpcProxy.format(NULL_CONTROLLER, req);
+          .setNsInfo(PBHelper.convert(nsInfo));
+      if(nameServiceId != null) {
+        req.setNameServiceId(nameServiceId);
+      }
+
+      rpcProxy.format(NULL_CONTROLLER, req.build());
     } catch (ServiceException e) {
       throw ProtobufHelper.getRemoteException(e);
     }
   }
 
   @Override
-  public NewEpochResponseProto newEpoch(String jid, NamespaceInfo nsInfo,
-      long epoch) throws IOException {
+  public NewEpochResponseProto newEpoch(String jid,
+                                        String nameServiceId,
+                                        NamespaceInfo nsInfo,
+                                        long epoch) throws IOException {
     try {
-      NewEpochRequestProto req = NewEpochRequestProto.newBuilder()
-        .setJid(convertJournalId(jid))
-        .setNsInfo(PBHelper.convert(nsInfo))
-        .setEpoch(epoch)
-        .build();
-      return rpcProxy.newEpoch(NULL_CONTROLLER, req);
+      NewEpochRequestProto.Builder req = NewEpochRequestProto.newBuilder()
+          .setJid(convertJournalId(jid))
+          .setNsInfo(PBHelper.convert(nsInfo))
+          .setEpoch(epoch);
+
+      if(nameServiceId != null) {
+        req.setNameServiceId(nameServiceId);
+      }
+
+      return rpcProxy.newEpoch(NULL_CONTROLLER, req.build());
     } catch (ServiceException e) {
       throw ProtobufHelper.getRemoteException(e);
     }
@@ -191,6 +210,9 @@ public class QJournalProtocolTranslatorPB implements ProtocolMetaInterface,
     if (reqInfo.hasCommittedTxId()) {
       builder.setCommittedTxId(reqInfo.getCommittedTxId());
     }
+    if(reqInfo.getNameServiceId() != null) {
+      builder.setNameServiceId(reqInfo.getNameServiceId());
+    }
     return builder.build();
   }
 
@@ -239,16 +261,21 @@ public class QJournalProtocolTranslatorPB implements ProtocolMetaInterface,
   }
 
   @Override
-  public GetEditLogManifestResponseProto getEditLogManifest(String jid,
-      long sinceTxId, boolean inProgressOk)
-      throws IOException {
+  public GetEditLogManifestResponseProto getEditLogManifest(
+      String jid, String nameServiceId,
+       long sinceTxId, boolean inProgressOk) throws IOException {
     try {
+      GetEditLogManifestRequestProto.Builder req;
+      req = GetEditLogManifestRequestProto.newBuilder()
+          .setJid(convertJournalId(jid))
+          .setSinceTxId(sinceTxId)
+          .setInProgressOk(inProgressOk);
+      if (nameServiceId !=null) {
+        req.setNameServiceId(nameServiceId);
+      }
       return rpcProxy.getEditLogManifest(NULL_CONTROLLER,
-          GetEditLogManifestRequestProto.newBuilder()
-            .setJid(convertJournalId(jid))
-            .setSinceTxId(sinceTxId)
-            .setInProgressOk(inProgressOk)
-            .build());
+          req.build()
+          );
     } catch (ServiceException e) {
       throw ProtobufHelper.getRemoteException(e);
     }
@@ -292,10 +319,10 @@ public class QJournalProtocolTranslatorPB implements ProtocolMetaInterface,
   @Override
   public void doPreUpgrade(String jid) throws IOException {
     try {
-      rpcProxy.doPreUpgrade(NULL_CONTROLLER,
-          DoPreUpgradeRequestProto.newBuilder()
-            .setJid(convertJournalId(jid))
-            .build());
+      DoPreUpgradeRequestProto.Builder req;
+      req = DoPreUpgradeRequestProto.newBuilder()
+          .setJid(convertJournalId(jid));
+      rpcProxy.doPreUpgrade(NULL_CONTROLLER, req.build());
     } catch (ServiceException e) {
       throw ProtobufHelper.getRemoteException(e);
     }
@@ -315,29 +342,37 @@ public class QJournalProtocolTranslatorPB implements ProtocolMetaInterface,
   }
   
   @Override
-  public void doFinalize(String jid) throws IOException {
+  public void doFinalize(String jid, String nameServiceId) throws IOException {
     try {
-      rpcProxy.doFinalize(NULL_CONTROLLER,
-          DoFinalizeRequestProto.newBuilder()
-            .setJid(convertJournalId(jid))
-            .build());
+      DoFinalizeRequestProto.Builder req = DoFinalizeRequestProto
+          .newBuilder()
+          .setJid(convertJournalId(jid));
+      if (nameServiceId != null) {
+        req.setNameServiceId(nameServiceId);
+      }
+      rpcProxy.doFinalize(NULL_CONTROLLER, req.build());
     } catch (ServiceException e) {
       throw ProtobufHelper.getRemoteException(e);
     }
   }
 
   @Override
-  public Boolean canRollBack(String journalId, StorageInfo storage,
-      StorageInfo prevStorage, int targetLayoutVersion) throws IOException {
+  public Boolean canRollBack(String journalId,
+                             String nameServiceId,
+                             StorageInfo storage,
+                             StorageInfo prevStorage,
+                             int targetLayoutVersion) throws IOException {
     try {
+      CanRollBackRequestProto.Builder req = CanRollBackRequestProto.newBuilder()
+          .setJid(convertJournalId(journalId))
+          .setStorage(PBHelper.convert(storage))
+          .setPrevStorage(PBHelper.convert(prevStorage))
+          .setTargetLayoutVersion(targetLayoutVersion);
+      if (nameServiceId != null) {
+        req.setNameServiceId(nameServiceId);
+      }
       CanRollBackResponseProto response = rpcProxy.canRollBack(
-          NULL_CONTROLLER,
-          CanRollBackRequestProto.newBuilder()
-            .setJid(convertJournalId(journalId))
-            .setStorage(PBHelper.convert(storage))
-            .setPrevStorage(PBHelper.convert(prevStorage))
-            .setTargetLayoutVersion(targetLayoutVersion)
-            .build());
+          NULL_CONTROLLER, req.build());
       return response.getCanRollBack();
     } catch (ServiceException e) {
       throw ProtobufHelper.getRemoteException(e);
@@ -345,38 +380,53 @@ public class QJournalProtocolTranslatorPB implements ProtocolMetaInterface,
   }
 
   @Override
-  public void doRollback(String journalId) throws IOException {
+  public void doRollback(String journalId,
+                         String nameServiceId) throws IOException {
     try {
-      rpcProxy.doRollback(NULL_CONTROLLER,
-          DoRollbackRequestProto.newBuilder()
-            .setJid(convertJournalId(journalId))
-            .build());
+      DoRollbackRequestProto.Builder req = DoRollbackRequestProto.newBuilder()
+          .setJid(convertJournalId(journalId));
+
+      if (nameServiceId != null) {
+        req.setNameserviceId(nameServiceId);
+      }
+      rpcProxy.doRollback(NULL_CONTROLLER, req.build());
     } catch (ServiceException e) {
       throw ProtobufHelper.getRemoteException(e);
     }
   }
 
   @Override
-  public void discardSegments(String journalId, long startTxId)
+  public void discardSegments(String journalId,
+                              String nameServiceId,
+                              long startTxId)
       throws IOException {
     try {
-      rpcProxy.discardSegments(NULL_CONTROLLER,
-          DiscardSegmentsRequestProto.newBuilder()
-            .setJid(convertJournalId(journalId)).setStartTxId(startTxId)
-            .build());
+      DiscardSegmentsRequestProto.Builder req = DiscardSegmentsRequestProto
+          .newBuilder()
+          .setJid(convertJournalId(journalId)).setStartTxId(startTxId);
+
+      if (nameServiceId != null) {
+        req.setNameServiceId(nameServiceId);
+      }
+      rpcProxy.discardSegments(NULL_CONTROLLER, req.build());
     } catch (ServiceException e) {
       throw ProtobufHelper.getRemoteException(e);
     }
   }
 
   @Override
-  public Long getJournalCTime(String journalId) throws IOException {
+  public Long getJournalCTime(String journalId,
+                              String nameServiceId) throws IOException {
     try {
+
+      GetJournalCTimeRequestProto.Builder req = GetJournalCTimeRequestProto
+          .newBuilder()
+          .setJid(convertJournalId(journalId));
+      if(nameServiceId !=null) {
+        req.setNameServiceId(nameServiceId);
+      }
       GetJournalCTimeResponseProto response = rpcProxy.getJournalCTime(
-          NULL_CONTROLLER,
-          GetJournalCTimeRequestProto.newBuilder()
-            .setJid(convertJournalId(journalId))
-            .build());
+          NULL_CONTROLLER, req.build());
       return response.getResultCTime();
     } catch (ServiceException e) {
       throw ProtobufHelper.getRemoteException(e);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java

@@ -137,6 +137,11 @@ public class Journal implements Closeable {
 
   private long lastJournalTimestamp = 0;
 
+  // This variable tracks, have we tried to start journalsyncer
+  // with nameServiceId. This will help not to start the journalsyncer
+  // on each rpc call, if it has failed to start
+  private boolean triedJournalSyncerStartedwithnsId = false;
+
   /**
    * Time threshold for sync calls, beyond which a warning should be logged to the console.
    */
@@ -160,6 +165,14 @@ public class Journal implements Closeable {
     }
   }
 
+  public void setTriedJournalSyncerStartedwithnsId(boolean started) {
+    this.triedJournalSyncerStartedwithnsId = started;
+  }
+
+  public boolean getTriedJournalSyncerStartedwithnsId() {
+    return triedJournalSyncerStartedwithnsId;
+  }
+
   /**
    * Reload any data that may have been cached. This is necessary
    * when we first load the Journal, but also after any formatting
@@ -660,7 +673,7 @@ public class Journal implements Closeable {
   }
 
   /**
-   * @see QJournalProtocol#getEditLogManifest(String, long, boolean)
+   * @see QJournalProtocol#getEditLogManifest(String, String, long, boolean)
    */
   public RemoteEditLogManifest getEditLogManifest(long sinceTxId,
       boolean inProgressOk) throws IOException {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java

@@ -86,7 +86,9 @@ public class JournalNode implements Tool, Configurable, JournalNodeMXBean {
    */
   private int resultCode = 0;
 
-  synchronized Journal getOrCreateJournal(String jid, StartupOption startOpt)
+  synchronized Journal getOrCreateJournal(String jid,
+                                          String nameServiceId,
+                                          StartupOption startOpt)
       throws IOException {
     QuorumJournalManager.checkJournalId(jid);
     
@@ -101,22 +103,46 @@ public class JournalNode implements Tool, Configurable, JournalNodeMXBean {
       if (conf.getBoolean(
           DFSConfigKeys.DFS_JOURNALNODE_ENABLE_SYNC_KEY,
           DFSConfigKeys.DFS_JOURNALNODE_ENABLE_SYNC_DEFAULT)) {
-        startSyncer(journal, jid);
+        startSyncer(journal, jid, nameServiceId);
       }
+    } else if (journalSyncersById.get(jid) != null &&
+        !journalSyncersById.get(jid).isJournalSyncerStarted() &&
+        !journalsById.get(jid).getTriedJournalSyncerStartedwithnsId() &&
+        nameServiceId != null) {
+      startSyncer(journal, jid, nameServiceId);
     }
 
+
     return journal;
   }
 
-  private void startSyncer(Journal journal, String jid) {
-    JournalNodeSyncer jSyncer = new JournalNodeSyncer(this, journal, jid, conf);
-    journalSyncersById.put(jid, jSyncer);
-    jSyncer.start();
+  @VisibleForTesting
+  public boolean getJournalSyncerStatus(String jid) {
+    if (journalSyncersById.get(jid) != null) {
+      return journalSyncersById.get(jid).isJournalSyncerStarted();
+    } else {
+      return false;
+    }
+  }
+
+  private void startSyncer(Journal journal, String jid, String nameServiceId) {
+    JournalNodeSyncer jSyncer = journalSyncersById.get(jid);
+    if (jSyncer == null) {
+      jSyncer = new JournalNodeSyncer(this, journal, jid, conf, nameServiceId);
+      journalSyncersById.put(jid, jSyncer);
+    }
+    jSyncer.start(nameServiceId);
   }
 
   @VisibleForTesting
   public Journal getOrCreateJournal(String jid) throws IOException {
-    return getOrCreateJournal(jid, StartupOption.REGULAR);
+    return getOrCreateJournal(jid, null, StartupOption.REGULAR);
+  }
+
+  public Journal getOrCreateJournal(String jid,
+                                    String nameServiceId)
+      throws IOException {
+    return getOrCreateJournal(jid, nameServiceId, StartupOption.REGULAR);
   }
 
   @Override
@@ -357,26 +383,40 @@ public class JournalNode implements Tool, Configurable, JournalNodeMXBean {
     getOrCreateJournal(journalId).doUpgrade(sInfo);
   }
 
-  public void doFinalize(String journalId) throws IOException {
-    getOrCreateJournal(journalId).doFinalize();
+  public void doFinalize(String journalId,
+                         String nameServiceId)
+      throws IOException {
+    getOrCreateJournal(journalId, nameServiceId).doFinalize();
   }
 
   public Boolean canRollBack(String journalId, StorageInfo storage,
-      StorageInfo prevStorage, int targetLayoutVersion) throws IOException {
-    return getOrCreateJournal(journalId, StartupOption.ROLLBACK).canRollBack(
+      StorageInfo prevStorage, int targetLayoutVersion,
+      String nameServiceId) throws IOException {
+    return getOrCreateJournal(journalId,
+        nameServiceId, StartupOption.ROLLBACK).canRollBack(
         storage, prevStorage, targetLayoutVersion);
   }
 
-  public void doRollback(String journalId) throws IOException {
-    getOrCreateJournal(journalId, StartupOption.ROLLBACK).doRollback();
+  public void doRollback(String journalId,
+                         String nameServiceId) throws IOException {
+    getOrCreateJournal(journalId,
+        nameServiceId, StartupOption.ROLLBACK).doRollback();
   }
 
-  public void discardSegments(String journalId, long startTxId)
+  public void discardSegments(String journalId, long startTxId,
+                              String nameServiceId)
       throws IOException {
-    getOrCreateJournal(journalId).discardSegments(startTxId);
+    getOrCreateJournal(journalId, nameServiceId).discardSegments(startTxId);
   }
 
-  public Long getJournalCTime(String journalId) throws IOException {
-    return getOrCreateJournal(journalId).getJournalCTime();
+  public Long getJournalCTime(String journalId,
+                              String nameServiceId) throws IOException {
+    return getOrCreateJournal(journalId, nameServiceId).getJournalCTime();
   }
+
+  @VisibleForTesting
+  public Journal getJournal(String  jid) {
+    return journalsById.get(jid);
+  }
+
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java

@@ -49,10 +49,10 @@ import org.apache.hadoop.net.NetUtils;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.protobuf.BlockingService;
 
+
 @InterfaceAudience.Private
 @VisibleForTesting
 public class JournalNodeRpcServer implements QJournalProtocol {
-
   private static final int HANDLER_COUNT = 5;
   private final JournalNode jn;
   private Server server;
@@ -117,15 +117,18 @@ public class JournalNodeRpcServer implements QJournalProtocol {
   }
 
   @Override
-  public boolean isFormatted(String journalId) throws IOException {
-    return jn.getOrCreateJournal(journalId).isFormatted();
+  public boolean isFormatted(String journalId,
+                             String nameServiceId) throws IOException {
+    return jn.getOrCreateJournal(journalId, nameServiceId).isFormatted();
   }
 
   @SuppressWarnings("deprecation")
   @Override
-  public GetJournalStateResponseProto getJournalState(String journalId)
+  public GetJournalStateResponseProto getJournalState(String journalId,
+                                                      String nameServiceId)
         throws IOException {
-    long epoch = jn.getOrCreateJournal(journalId).getLastPromisedEpoch(); 
+    long epoch = jn.getOrCreateJournal(journalId,
+        nameServiceId).getLastPromisedEpoch();
     return GetJournalStateResponseProto.newBuilder()
         .setLastPromisedEpoch(epoch)
         .setHttpPort(jn.getBoundHttpAddress().getPort())
@@ -135,59 +138,64 @@ public class JournalNodeRpcServer implements QJournalProtocol {
 
   @Override
   public NewEpochResponseProto newEpoch(String journalId,
-      NamespaceInfo nsInfo,
+                                        String nameServiceId,
+                                        NamespaceInfo nsInfo,
       long epoch) throws IOException {
-    return jn.getOrCreateJournal(journalId).newEpoch(nsInfo, epoch);
+    return jn.getOrCreateJournal(journalId,
+        nameServiceId).newEpoch(nsInfo, epoch);
   }
 
   @Override
-  public void format(String journalId, NamespaceInfo nsInfo)
+  public void format(String journalId,
+                     String nameServiceId,
+                     NamespaceInfo nsInfo)
       throws IOException {
-    jn.getOrCreateJournal(journalId).format(nsInfo);
+    jn.getOrCreateJournal(journalId, nameServiceId).format(nsInfo);
   }
 
   @Override
   public void journal(RequestInfo reqInfo,
       long segmentTxId, long firstTxnId,
       int numTxns, byte[] records) throws IOException {
-    jn.getOrCreateJournal(reqInfo.getJournalId())
+    jn.getOrCreateJournal(reqInfo.getJournalId(), reqInfo.getNameServiceId())
        .journal(reqInfo, segmentTxId, firstTxnId, numTxns, records);
   }
   
   @Override
   public void heartbeat(RequestInfo reqInfo) throws IOException {
-    jn.getOrCreateJournal(reqInfo.getJournalId())
+    jn.getOrCreateJournal(reqInfo.getJournalId(), reqInfo.getNameServiceId())
       .heartbeat(reqInfo);
   }
 
   @Override
   public void startLogSegment(RequestInfo reqInfo, long txid, int layoutVersion)
       throws IOException {
-    jn.getOrCreateJournal(reqInfo.getJournalId())
+    jn.getOrCreateJournal(reqInfo.getJournalId(), reqInfo.getNameServiceId())
       .startLogSegment(reqInfo, txid, layoutVersion);
   }
 
   @Override
   public void finalizeLogSegment(RequestInfo reqInfo, long startTxId,
       long endTxId) throws IOException {
-    jn.getOrCreateJournal(reqInfo.getJournalId())
+    jn.getOrCreateJournal(reqInfo.getJournalId(), reqInfo.getNameServiceId())
       .finalizeLogSegment(reqInfo, startTxId, endTxId);
   }
 
   @Override
   public void purgeLogsOlderThan(RequestInfo reqInfo, long minTxIdToKeep)
       throws IOException {
-    jn.getOrCreateJournal(reqInfo.getJournalId())
+    jn.getOrCreateJournal(reqInfo.getJournalId(), reqInfo.getNameServiceId())
       .purgeLogsOlderThan(reqInfo, minTxIdToKeep);
   }
 
   @SuppressWarnings("deprecation")
   @Override
-  public GetEditLogManifestResponseProto getEditLogManifest(String jid,
+  public GetEditLogManifestResponseProto getEditLogManifest(
+      String jid, String nameServiceId,
       long sinceTxId, boolean inProgressOk)
       throws IOException {
     
-    RemoteEditLogManifest manifest = jn.getOrCreateJournal(jid)
+    RemoteEditLogManifest manifest = jn.getOrCreateJournal(jid, nameServiceId)
         .getEditLogManifest(sinceTxId, inProgressOk);
     
     return GetEditLogManifestResponseProto.newBuilder()
@@ -200,14 +208,15 @@ public class JournalNodeRpcServer implements QJournalProtocol {
   @Override
   public PrepareRecoveryResponseProto prepareRecovery(RequestInfo reqInfo,
       long segmentTxId) throws IOException {
-    return jn.getOrCreateJournal(reqInfo.getJournalId())
+    return jn.getOrCreateJournal(reqInfo.getJournalId(),
+        reqInfo.getNameServiceId())
         .prepareRecovery(reqInfo, segmentTxId);
   }
 
   @Override
   public void acceptRecovery(RequestInfo reqInfo, SegmentStateProto log,
       URL fromUrl) throws IOException {
-    jn.getOrCreateJournal(reqInfo.getJournalId())
+    jn.getOrCreateJournal(reqInfo.getJournalId(), reqInfo.getNameServiceId())
         .acceptRecovery(reqInfo, log, fromUrl);
   }
 
@@ -222,30 +231,36 @@ public class JournalNodeRpcServer implements QJournalProtocol {
   }
 
   @Override
-  public void doFinalize(String journalId) throws IOException {
-    jn.doFinalize(journalId);
+  public void doFinalize(String journalId,
+                         String nameServiceId) throws IOException {
+    jn.doFinalize(journalId, nameServiceId);
   }
 
   @Override
-  public Boolean canRollBack(String journalId, StorageInfo storage,
+  public Boolean canRollBack(String journalId,
+                             String nameServiceId, StorageInfo storage,
       StorageInfo prevStorage, int targetLayoutVersion)
       throws IOException {
-    return jn.canRollBack(journalId, storage, prevStorage, targetLayoutVersion);
+    return jn.canRollBack(journalId, storage, prevStorage, targetLayoutVersion,
+        nameServiceId);
   }
 
   @Override
-  public void doRollback(String journalId) throws IOException {
-    jn.doRollback(journalId);
+  public void doRollback(String journalId,
+                         String nameServiceId) throws IOException {
+    jn.doRollback(journalId, nameServiceId);
   }
 
   @Override
-  public void discardSegments(String journalId, long startTxId)
+  public void discardSegments(String journalId,
+                              String nameServiceId, long startTxId)
       throws IOException {
-    jn.discardSegments(journalId, startTxId);
+    jn.discardSegments(journalId, startTxId, nameServiceId);
   }
 
   @Override
-  public Long getJournalCTime(String journalId) throws IOException {
-    return jn.getJournalCTime(journalId);
+  public Long getJournalCTime(String journalId,
+                              String nameServiceId) throws IOException {
+    return jn.getJournalCTime(journalId, nameServiceId);
   }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java

@@ -65,6 +65,7 @@ public class JournalNodeSyncer {
   private final JournalNode jn;
   private final Journal journal;
   private final String jid;
+  private  String nameServiceId;
   private final JournalIdProto jidProto;
   private final JNStorage jnStorage;
   private final Configuration conf;
@@ -78,12 +79,14 @@ public class JournalNodeSyncer {
   private final int logSegmentTransferTimeout;
   private final DataTransferThrottler throttler;
   private final JournalMetrics metrics;
+  private boolean journalSyncerStarted;
 
   JournalNodeSyncer(JournalNode jouranlNode, Journal journal, String jid,
-      Configuration conf) {
+      Configuration conf, String nameServiceId) {
     this.jn = jouranlNode;
     this.journal = journal;
     this.jid = jid;
+    this.nameServiceId = nameServiceId;
     this.jidProto = convertJournalId(this.jid);
     this.jnStorage = journal.getStorage();
     this.conf = conf;
@@ -95,6 +98,7 @@ public class JournalNodeSyncer {
         DFSConfigKeys.DFS_EDIT_LOG_TRANSFER_TIMEOUT_DEFAULT);
     throttler = getThrottler(conf);
     metrics = journal.getMetrics();
+    journalSyncerStarted = false;
   }
 
   void stopSync() {
@@ -109,13 +113,21 @@ public class JournalNodeSyncer {
     }
   }
 
-  public void start() {
-    LOG.info("Starting SyncJournal daemon for journal " + jid);
-    if (getOtherJournalNodeProxies()) {
+  public void start(String nsId) {
+    if (nsId != null) {
+      this.nameServiceId = nsId;
+      journal.setTriedJournalSyncerStartedwithnsId(true);
+    }
+    if (!journalSyncerStarted && getOtherJournalNodeProxies()) {
+      LOG.info("Starting SyncJournal daemon for journal " + jid);
       startSyncJournalsDaemon();
-    } else {
-      LOG.warn("Failed to start SyncJournal daemon for journal " + jid);
+      journalSyncerStarted = true;
     }
+
+  }
+
+  public boolean isJournalSyncerStarted() {
+    return journalSyncerStarted;
   }
 
   private boolean createEditsSyncDir() {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java

@@ -833,10 +833,12 @@ public class BlockManager implements BlockStatsMXBean {
     // l: == live:, d: == decommissioned c: == corrupt e: == excess
     out.print(block + ((usableReplicas > 0)? "" : " MISSING") +
               " (replicas:" +
-              " l: " + numReplicas.liveReplicas() +
-              " d: " + numReplicas.decommissionedAndDecommissioning() +
-              " c: " + numReplicas.corruptReplicas() +
-              " e: " + numReplicas.excessReplicas() + ") ");
+              " live: " + numReplicas.liveReplicas() +
+              " decommissioning and decommissioned: " +
+        numReplicas.decommissionedAndDecommissioning() +
+              " corrupt: " + numReplicas.corruptReplicas() +
+              " in excess: " + numReplicas.excessReplicas() +
+              " maintenance mode: " + numReplicas.maintenanceReplicas() + ") ");
 
     Collection<DatanodeDescriptor> corruptNodes = 
                                   corruptReplicas.getNodes(block);
@@ -849,6 +851,8 @@ public class BlockManager implements BlockStatsMXBean {
       } else if (node.isDecommissioned() || 
           node.isDecommissionInProgress()) {
         state = "(decommissioned)";
+      } else if (node.isMaintenance() || node.isInMaintenance()){
+        state = "(maintenance)";
       }
       
       if (storage.areBlockContentsStale()) {
@@ -3889,11 +3893,15 @@ public class BlockManager implements BlockStatsMXBean {
       throw new IOException(
           "Got incremental block report from unregistered or dead node");
     }
+
+    boolean successful = false;
     try {
       processIncrementalBlockReport(node, srdb);
-    } catch (Exception ex) {
-      node.setForceRegistration(true);
-      throw ex;
+      successful = true;
+    } finally {
+      if (!successful) {
+        node.setForceRegistration(true);
+      }
     }
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java

@@ -197,10 +197,9 @@ public class BlockRecoveryWorker {
       long blockId = (isTruncateRecovery) ?
           rBlock.getNewBlock().getBlockId() : block.getBlockId();
 
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("block=" + block + ", (length=" + block.getNumBytes()
-            + "), syncList=" + syncList);
-      }
+      LOG.info("BlockRecoveryWorker: block={} (length={}),"
+              + " isTruncateRecovery={}, syncList={}", block,
+          block.getNumBytes(), isTruncateRecovery, syncList);
 
       // syncList.isEmpty() means that all data-nodes do not have the block
       // or their replicas have 0 length.
@@ -289,6 +288,11 @@ public class BlockRecoveryWorker {
         newBlock.setNumBytes(rBlock.getNewBlock().getNumBytes());
       }
 
+      LOG.info("BlockRecoveryWorker: block={} (length={}), bestState={},"
+              + " newBlock={} (length={}), participatingList={}",
+          block, block.getNumBytes(), bestState.name(), newBlock,
+          newBlock.getNumBytes(), participatingList);
+
       List<DatanodeID> failedList = new ArrayList<>();
       final List<BlockRecord> successList = new ArrayList<>();
       for (BlockRecord r : participatingList) {
@@ -542,7 +546,7 @@ public class BlockRecoveryWorker {
     ExtendedBlock block = rb.getBlock();
     DatanodeInfo[] targets = rb.getLocations();
 
-    LOG.info(who + " calls recoverBlock(" + block
+    LOG.info("BlockRecoveryWorker: " + who + " calls recoverBlock(" + block
         + ", targets=[" + Joiner.on(", ").join(targets) + "]"
         + ", newGenerationStamp=" + rb.getNewGenerationStamp()
         + ", newBlock=" + rb.getNewBlock()

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java

@@ -155,7 +155,7 @@ public class StripedBlockChecksumReconstructor extends StripedReconstructor {
     return checksumBuf.length;
   }
 
-  private void reconstructTargets(int toReconstructLen) {
+  private void reconstructTargets(int toReconstructLen) throws IOException {
     ByteBuffer[] inputs = getStripedReader().getInputBuffers(toReconstructLen);
 
     ByteBuffer[] outputs = new ByteBuffer[1];

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java

@@ -113,7 +113,7 @@ class StripedBlockReconstructor extends StripedReconstructor
     }
   }
 
-  private void reconstructTargets(int toReconstructLen) {
+  private void reconstructTargets(int toReconstructLen) throws IOException {
     ByteBuffer[] inputs = getStripedReader().getInputBuffers(toReconstructLen);
 
     int[] erasedIndices = stripedWriter.getRealTargetIndices();

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java

@@ -253,6 +253,14 @@ public final class ErasureCodingPolicyManager {
         return p;
       }
     }
+
+    if (getCurrentMaxPolicyID() == ErasureCodeConstants.MAX_POLICY_ID) {
+      throw new HadoopIllegalArgumentException("Adding erasure coding " +
+          "policy failed because the number of policies stored in the " +
+          "system already reached the threshold, which is " +
+          ErasureCodeConstants.MAX_POLICY_ID);
+    }
+
     policy.setName(assignedNewName);
     policy.setId(getNextAvailablePolicyID());
     this.policiesByName.put(policy.getName(), policy);
@@ -261,12 +269,14 @@ public final class ErasureCodingPolicyManager {
     return policy;
   }
 
+  private byte getCurrentMaxPolicyID() {
+    return policiesByID.keySet().stream().max(Byte::compareTo).orElse((byte)0);
+  }
+
   private byte getNextAvailablePolicyID() {
-    byte currentId = this.policiesByID.keySet().stream()
-        .max(Byte::compareTo)
-        .filter(id -> id >= ErasureCodeConstants.USER_DEFINED_POLICY_START_ID)
-        .orElse(ErasureCodeConstants.USER_DEFINED_POLICY_START_ID);
-    return (byte) (currentId + 1);
+    byte nextPolicyID = (byte)(getCurrentMaxPolicyID() + 1);
+    return nextPolicyID > ErasureCodeConstants.USER_DEFINED_POLICY_START_ID ?
+        nextPolicyID : ErasureCodeConstants.USER_DEFINED_POLICY_START_ID;
   }
 
   /**

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java

@@ -380,10 +380,16 @@ final class FSDirEncryptionZoneOp {
   static void saveFileXAttrsForBatch(FSDirectory fsd,
       List<FileEdekInfo> batch) {
     assert fsd.getFSNamesystem().hasWriteLock();
+    assert !fsd.hasWriteLock();
     if (batch != null && !batch.isEmpty()) {
       for (FileEdekInfo entry : batch) {
         final INode inode = fsd.getInode(entry.getInodeId());
-        Preconditions.checkNotNull(inode);
+        // no dir lock, so inode could be removed. no-op if so.
+        if (inode == null) {
+          NameNode.LOG.info("Cannot find inode {}, skip saving xattr for"
+              + " re-encryption", entry.getInodeId());
+          continue;
+        }
         fsd.getEditLog().logSetXAttrs(inode.getFullPathName(),
             inode.getXAttrFeature().getXAttrs(), false);
       }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java

@@ -130,7 +130,7 @@ public class FSEditLog implements LogsPurgeable {
    * 
    * In a non-HA setup:
    * 
-   * The log starts in UNITIALIZED state upon construction. Once it's
+   * The log starts in UNINITIALIZED state upon construction. Once it's
    * initialized, it is usually in IN_SEGMENT state, indicating that edits may
    * be written. In the middle of a roll, or while saving the namespace, it
    * briefly enters the BETWEEN_LOG_SEGMENTS state, indicating that the previous
@@ -1805,8 +1805,20 @@ public class FSEditLog implements LogsPurgeable {
     try {
       Constructor<? extends JournalManager> cons
         = clazz.getConstructor(Configuration.class, URI.class,
+            NamespaceInfo.class, String.class);
+      String nameServiceId = conf.get(DFSConfigKeys.DFS_NAMESERVICE_ID);
+      return cons.newInstance(conf, uri, storage.getNamespaceInfo(),
+          nameServiceId);
+    } catch (NoSuchMethodException ne) {
+      try {
+        Constructor<? extends JournalManager> cons
+            = clazz.getConstructor(Configuration.class, URI.class,
             NamespaceInfo.class);
-      return cons.newInstance(conf, uri, storage.getNamespaceInfo());
+        return cons.newInstance(conf, uri, storage.getNamespaceInfo());
+      } catch (Exception e) {
+        throw new IllegalArgumentException("Unable to construct journal, "
+            + uri, e);
+      }
     } catch (Exception e) {
       throw new IllegalArgumentException("Unable to construct journal, "
                                          + uri, e);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -3673,6 +3673,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
       readUnlock(operationName);
     }
     logAuditEvent(true, operationName, src);
+    if (topConf.isEnabled && isAuditEnabled() && isExternalInvocation()
+        && dl != null && Server.getRemoteUser() != null) {
+      topMetrics.reportFilesInGetListing(Server.getRemoteUser().toString(),
+          dl.getPartialListing().length);
+    }
     return dl;
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java

@@ -275,8 +275,16 @@ class FSPermissionChecker implements AccessControlEnforcer {
     INodeAttributes inodeAttrs = inode.getSnapshotINode(snapshotId);
     if (getAttributesProvider() != null) {
       String[] elements = new String[pathIdx + 1];
-      for (int i = 0; i < elements.length; i++) {
-        elements[i] = DFSUtil.bytes2String(pathByNameArr[i]);
+      /**
+       * {@link INode#getPathComponents(String)} returns a null component
+       * for the root only path "/". Assign an empty string if so.
+       */
+      if (pathByNameArr.length == 1 && pathByNameArr[0] == null) {
+        elements[0] = "";
+      } else {
+        for (int i = 0; i < elements.length; i++) {
+          elements[i] = DFSUtil.bytes2String(pathByNameArr[i]);
+        }
       }
       inodeAttrs = getAttributesProvider().getAttributes(elements, inodeAttrs);
     }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java

@@ -37,8 +37,6 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
 import com.google.common.collect.Lists;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -50,6 +48,8 @@ import org.apache.hadoop.util.Daemon;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * LeaseManager does the lease housekeeping for writing on files.   
@@ -75,7 +75,8 @@ import org.apache.hadoop.util.Time;
  */
 @InterfaceAudience.Private
 public class LeaseManager {
-  public static final Log LOG = LogFactory.getLog(LeaseManager.class);
+  public static final Logger LOG = LoggerFactory.getLogger(LeaseManager.class
+      .getName());
   private final FSNamesystem fsnamesystem;
   private long softLimit = HdfsConstants.LEASE_SOFTLIMIT_PERIOD;
   private long hardLimit = HdfsConstants.LEASE_HARDLIMIT_PERIOD;
@@ -142,8 +143,8 @@ public class LeaseManager {
     for (Long id : getINodeIdWithLeases()) {
       final INodeFile cons = fsnamesystem.getFSDirectory().getInode(id).asFile();
       if (!cons.isUnderConstruction()) {
-        LOG.warn("The file " + cons.getFullPathName()
-            + " is not under construction but has lease.");
+        LOG.warn("The file {} is not under construction but has lease.",
+            cons.getFullPathName());
         continue;
       }
       BlockInfo[] blocks = cons.getBlocks();
@@ -155,7 +156,7 @@ public class LeaseManager {
           numUCBlocks++;
         }
       }
-    LOG.info("Number of blocks under construction: " + numUCBlocks);
+    LOG.info("Number of blocks under construction: {}", numUCBlocks);
     return numUCBlocks;
   }
 
@@ -250,9 +251,8 @@ public class LeaseManager {
     }
     final long endTimeMs = Time.monotonicNow();
     if ((endTimeMs - startTimeMs) > 1000) {
-      LOG.info("Took " + (endTimeMs - startTimeMs) + " ms to collect "
-          + iipSet.size() + " open files with leases" +
-          ((ancestorDir != null) ?
+      LOG.info("Took {} ms to collect {} open files with leases {}",
+          (endTimeMs - startTimeMs), iipSet.size(), ((ancestorDir != null) ?
               " under " + ancestorDir.getFullPathName() : "."));
     }
     return iipSet;
@@ -287,8 +287,8 @@ public class LeaseManager {
       final INodeFile inodeFile =
           fsnamesystem.getFSDirectory().getInode(inodeId).asFile();
       if (!inodeFile.isUnderConstruction()) {
-        LOG.warn("The file " + inodeFile.getFullPathName()
-            + " is not under construction but has lease.");
+        LOG.warn("The file {} is not under construction but has lease.",
+            inodeFile.getFullPathName());
         continue;
       }
       openFileEntries.add(new OpenFileEntry(
@@ -348,16 +348,13 @@ public class LeaseManager {
   private synchronized void removeLease(Lease lease, long inodeId) {
     leasesById.remove(inodeId);
     if (!lease.removeFile(inodeId)) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("inode " + inodeId + " not found in lease.files (=" + lease
-                + ")");
-      }
+      LOG.debug("inode {} not found in lease.files (={})", inodeId, lease);
     }
 
     if (!lease.hasFiles()) {
       leases.remove(lease.holder);
       if (!sortedLeases.remove(lease)) {
-        LOG.error(lease + " not found in sortedLeases");
+        LOG.error("{} not found in sortedLeases", lease);
       }
     }
   }
@@ -370,8 +367,8 @@ public class LeaseManager {
     if (lease != null) {
       removeLease(lease, src.getId());
     } else {
-      LOG.warn("Removing non-existent lease! holder=" + holder +
-          " src=" + src.getFullPathName());
+      LOG.warn("Removing non-existent lease! holder={} src={}", holder, src
+          .getFullPathName());
     }
   }
 
@@ -513,9 +510,7 @@ public class LeaseManager {
   
           Thread.sleep(fsnamesystem.getLeaseRecheckIntervalMs());
         } catch(InterruptedException ie) {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug(name + " is interrupted", ie);
-          }
+          LOG.debug("{} is interrupted", name, ie);
         } catch(Throwable e) {
           LOG.warn("Unexpected throwable: ", e);
         }
@@ -537,7 +532,7 @@ public class LeaseManager {
         sortedLeases.first().expiredHardLimit()
         && !isMaxLockHoldToReleaseLease(start)) {
       Lease leaseToCheck = sortedLeases.first();
-      LOG.info(leaseToCheck + " has expired hard limit");
+      LOG.info("{} has expired hard limit", leaseToCheck);
 
       final List<Long> removing = new ArrayList<>();
       // need to create a copy of the oldest lease files, because
@@ -568,16 +563,16 @@ public class LeaseManager {
             completed = fsnamesystem.internalReleaseLease(
                 leaseToCheck, p, iip, newHolder);
           } catch (IOException e) {
-            LOG.warn("Cannot release the path " + p + " in the lease "
-                + leaseToCheck + ". It will be retried.", e);
+            LOG.warn("Cannot release the path {} in the lease {}. It will be "
+                + "retried.", p, leaseToCheck, e);
             continue;
           }
           if (LOG.isDebugEnabled()) {
             if (completed) {
-              LOG.debug("Lease recovery for inode " + id + " is complete. " +
-                            "File closed.");
+              LOG.debug("Lease recovery for inode {} is complete. File closed"
+                  + ".", id);
             } else {
-              LOG.debug("Started block recovery " + p + " lease " + leaseToCheck);
+              LOG.debug("Started block recovery {} lease {}", p, leaseToCheck);
             }
           }
           // If a lease recovery happened, we need to sync later.
@@ -585,13 +580,13 @@ public class LeaseManager {
             needSync = true;
           }
         } catch (IOException e) {
-          LOG.warn("Removing lease with an invalid path: " + p + ","
-              + leaseToCheck, e);
+          LOG.warn("Removing lease with an invalid path: {},{}", p,
+              leaseToCheck, e);
           removing.add(id);
         }
         if (isMaxLockHoldToReleaseLease(start)) {
-          LOG.debug("Breaking out of checkLeases after " +
-              fsnamesystem.getMaxLockHoldToReleaseLeaseMs() + "ms.");
+          LOG.debug("Breaking out of checkLeases after {} ms.",
+              fsnamesystem.getMaxLockHoldToReleaseLeaseMs());
           break;
         }
       }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java

@@ -45,11 +45,11 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.security.GeneralSecurityException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorCompletionService;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -112,8 +112,7 @@ public class ReencryptionHandler implements Runnable {
   private ExecutorCompletionService<ReencryptionTask> batchService;
   private BlockingQueue<Runnable> taskQueue;
   // protected by ReencryptionHandler object lock
-  private final Map<Long, ZoneSubmissionTracker> submissions =
-      new ConcurrentHashMap<>();
+  private final Map<Long, ZoneSubmissionTracker> submissions = new HashMap<>();
 
   // The current batch that the handler is working on. Handler is designed to
   // be single-threaded, see class javadoc for more details.
@@ -132,8 +131,10 @@ public class ReencryptionHandler implements Runnable {
    */
   void stopThreads() {
     assert dir.hasWriteLock();
-    for (ZoneSubmissionTracker zst : submissions.values()) {
-      zst.cancelAllTasks();
+    synchronized (this) {
+      for (ZoneSubmissionTracker zst : submissions.values()) {
+        zst.cancelAllTasks();
+      }
     }
     if (updaterExecutor != null) {
       updaterExecutor.shutdownNow();
@@ -269,33 +270,34 @@ public class ReencryptionHandler implements Runnable {
       throw new IOException("Zone " + zoneName + " is not under re-encryption");
     }
     zs.cancel();
-    ZoneSubmissionTracker zst = submissions.get(zoneId);
-    if (zst != null) {
-      zst.cancelAllTasks();
-    }
+    removeZoneTrackerStopTasks(zoneId);
   }
 
   void removeZone(final long zoneId) {
     assert dir.hasWriteLock();
     LOG.info("Removing zone {} from re-encryption.", zoneId);
-    ZoneSubmissionTracker zst = submissions.get(zoneId);
+    removeZoneTrackerStopTasks(zoneId);
+    getReencryptionStatus().removeZone(zoneId);
+  }
+
+  synchronized private void removeZoneTrackerStopTasks(final long zoneId) {
+    final ZoneSubmissionTracker zst = submissions.get(zoneId);
     if (zst != null) {
       zst.cancelAllTasks();
+      submissions.remove(zoneId);
     }
-    submissions.remove(zoneId);
-    getReencryptionStatus().removeZone(zoneId);
   }
 
   ZoneSubmissionTracker getTracker(final long zoneId) {
-    dir.hasReadLock();
+    assert dir.hasReadLock();
     return unprotectedGetTracker(zoneId);
   }
 
   /**
-   * get the tracker without holding the FSDirectory lock. This is only used for
-   * testing, when updater checks about pausing.
+   * Get the tracker without holding the FSDirectory lock.
+   * The submissions object is protected by object lock.
    */
-  ZoneSubmissionTracker unprotectedGetTracker(final long zoneId) {
+  synchronized ZoneSubmissionTracker unprotectedGetTracker(final long zoneId) {
     return submissions.get(zoneId);
   }
 
@@ -308,16 +310,19 @@ public class ReencryptionHandler implements Runnable {
    *
    * @param zoneId
    */
-  void addDummyTracker(final long zoneId) {
+  void addDummyTracker(final long zoneId, ZoneSubmissionTracker zst) {
     assert dir.hasReadLock();
-    assert !submissions.containsKey(zoneId);
-    final ZoneSubmissionTracker zst = new ZoneSubmissionTracker();
+    if (zst == null) {
+      zst = new ZoneSubmissionTracker();
+    }
     zst.setSubmissionDone();
 
-    Future future = batchService.submit(
+    final Future future = batchService.submit(
         new EDEKReencryptCallable(zoneId, new ReencryptionBatch(), this));
     zst.addTask(future);
-    submissions.put(zoneId, zst);
+    synchronized (this) {
+      submissions.put(zoneId, zst);
+    }
   }
 
   /**
@@ -351,6 +356,8 @@ public class ReencryptionHandler implements Runnable {
         }
         LOG.info("Executing re-encrypt commands on zone {}. Current zones:{}",
             zoneId, getReencryptionStatus());
+        getReencryptionStatus().markZoneStarted(zoneId);
+        resetSubmissionTracker(zoneId);
       } finally {
         dir.readUnlock();
       }
@@ -392,7 +399,6 @@ public class ReencryptionHandler implements Runnable {
 
     readLock();
     try {
-      getReencryptionStatus().markZoneStarted(zoneId);
       zoneNode = dir.getInode(zoneId);
       // start re-encrypting the zone from the beginning
       if (zoneNode == null) {
@@ -428,6 +434,20 @@ public class ReencryptionHandler implements Runnable {
     }
   }
 
+  /**
+   * Reset the zone submission tracker for re-encryption.
+   * @param zoneId
+   */
+  synchronized private void resetSubmissionTracker(final long zoneId) {
+    ZoneSubmissionTracker zst = submissions.get(zoneId);
+    if (zst == null) {
+      zst = new ZoneSubmissionTracker();
+      submissions.put(zoneId, zst);
+    } else {
+      zst.reset();
+    }
+  }
+
   List<XAttr> completeReencryption(final INode zoneNode) throws IOException {
     assert dir.hasWriteLock();
     assert dir.getFSNamesystem().hasWriteLock();
@@ -437,8 +457,9 @@ public class ReencryptionHandler implements Runnable {
     LOG.info("Re-encryption completed on zone {}. Re-encrypted {} files,"
             + " failures encountered: {}.", zoneNode.getFullPathName(),
         zs.getFilesReencrypted(), zs.getNumReencryptionFailures());
-    // This also removes the zone from reencryptionStatus
-    submissions.remove(zoneId);
+    synchronized (this) {
+      submissions.remove(zoneId);
+    }
     return FSDirEncryptionZoneOp
         .updateReencryptionFinish(dir, INodesInPath.fromINode(zoneNode), zs);
   }
@@ -562,10 +583,13 @@ public class ReencryptionHandler implements Runnable {
     if (currentBatch.isEmpty()) {
       return;
     }
-    ZoneSubmissionTracker zst = submissions.get(zoneId);
-    if (zst == null) {
-      zst = new ZoneSubmissionTracker();
-      submissions.put(zoneId, zst);
+    ZoneSubmissionTracker zst;
+    synchronized (this) {
+      zst = submissions.get(zoneId);
+      if (zst == null) {
+        zst = new ZoneSubmissionTracker();
+        submissions.put(zoneId, zst);
+      }
     }
     Future future = batchService
         .submit(new EDEKReencryptCallable(zoneId, currentBatch, this));
@@ -821,19 +845,13 @@ public class ReencryptionHandler implements Runnable {
     // 2. if tasks are piling up on the updater, don't create new callables
     // until the queue size goes down.
     final int maxTasksPiled = Runtime.getRuntime().availableProcessors() * 2;
-    int totalTasks = 0;
-    for (ZoneSubmissionTracker zst : submissions.values()) {
-      totalTasks += zst.getTasks().size();
-    }
-    if (totalTasks >= maxTasksPiled) {
+    int numTasks = numTasksSubmitted();
+    if (numTasks >= maxTasksPiled) {
       LOG.debug("Re-encryption handler throttling because total tasks pending"
-          + " re-encryption updater is {}", totalTasks);
-      while (totalTasks >= maxTasksPiled) {
+          + " re-encryption updater is {}", numTasks);
+      while (numTasks >= maxTasksPiled) {
         Thread.sleep(500);
-        totalTasks = 0;
-        for (ZoneSubmissionTracker zst : submissions.values()) {
-          totalTasks += zst.getTasks().size();
-        }
+        numTasks = numTasksSubmitted();
       }
     }
 
@@ -864,6 +882,14 @@ public class ReencryptionHandler implements Runnable {
     throttleTimerLocked.reset();
   }
 
+  private synchronized int numTasksSubmitted() {
+    int ret = 0;
+    for (ZoneSubmissionTracker zst : submissions.values()) {
+      ret += zst.getTasks().size();
+    }
+    return ret;
+  }
+
   /**
    * Process an Inode for re-encryption. Add to current batch if it's a file,
    * no-op otherwise.
@@ -877,7 +903,7 @@ public class ReencryptionHandler implements Runnable {
    */
   private boolean reencryptINode(final INode inode, final String ezKeyVerName)
       throws IOException, InterruptedException {
-    dir.hasReadLock();
+    assert dir.hasReadLock();
     if (LOG.isTraceEnabled()) {
       LOG.trace("Processing {} for re-encryption", inode.getFullPathName());
     }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java

@@ -93,6 +93,13 @@ public final class ReencryptionUpdater implements Runnable {
       numFutureDone = 0;
     }
 
+    void reset() {
+      submissionDone = false;
+      tasks.clear();
+      numCheckpointed = 0;
+      numFutureDone = 0;
+    }
+
     LinkedList<Future> getTasks() {
       return tasks;
     }
@@ -238,12 +245,12 @@ public final class ReencryptionUpdater implements Runnable {
   void markZoneSubmissionDone(final long zoneId)
       throws IOException, InterruptedException {
     final ZoneSubmissionTracker tracker = handler.getTracker(zoneId);
-    if (tracker != null) {
+    if (tracker != null && !tracker.getTasks().isEmpty()) {
       tracker.submissionDone = true;
     } else {
       // Caller thinks submission is done, but no tasks submitted - meaning
       // no files in the EZ need to be re-encrypted. Complete directly.
-      handler.addDummyTracker(zoneId);
+      handler.addDummyTracker(zoneId, tracker);
     }
   }
 
@@ -289,6 +296,7 @@ public final class ReencryptionUpdater implements Runnable {
       LOG.debug(
           "Updating file xattrs for re-encrypting zone {}," + " starting at {}",
           zoneNodePath, task.batch.getFirstFilePath());
+      final int batchSize = task.batch.size();
       for (Iterator<FileEdekInfo> it = task.batch.getBatch().iterator();
            it.hasNext();) {
         FileEdekInfo entry = it.next();
@@ -342,7 +350,7 @@ public final class ReencryptionUpdater implements Runnable {
       }
 
       LOG.info("Updated xattrs on {}({}) files in zone {} for re-encryption,"
-              + " starting:{}.", task.numFilesUpdated, task.batch.size(),
+              + " starting:{}.", task.numFilesUpdated, batchSize,
           zoneNodePath, task.batch.getFirstFilePath());
     }
     task.processed = true;
@@ -377,6 +385,9 @@ public final class ReencryptionUpdater implements Runnable {
     ListIterator<Future> iter = tasks.listIterator();
     while (iter.hasNext()) {
       Future<ReencryptionTask> curr = iter.next();
+      if (curr.isCancelled()) {
+        break;
+      }
       if (!curr.isDone() || !curr.get().processed) {
         // still has earlier tasks not completed, skip here.
         break;
@@ -411,12 +422,12 @@ public final class ReencryptionUpdater implements Runnable {
     final Future<ReencryptionTask> completed = batchService.take();
     throttle();
     checkPauseForTesting();
-    ReencryptionTask task = completed.get();
     if (completed.isCancelled()) {
-      LOG.debug("Skipped canceled re-encryption task for zone {}, last: {}",
-          task.zoneId, task.lastFile);
+      // Ignore canceled zones. The cancellation is edit-logged by the handler.
+      LOG.debug("Skipped a canceled re-encryption task");
       return;
     }
+    final ReencryptionTask task = completed.get();
 
     boolean shouldRetry;
     do {
@@ -465,7 +476,11 @@ public final class ReencryptionUpdater implements Runnable {
           task.batch.size(), task.batch.getFirstFilePath());
       final ZoneSubmissionTracker tracker =
           handler.getTracker(zoneNode.getId());
-      Preconditions.checkNotNull(tracker, "zone tracker not found " + zonePath);
+      if (tracker == null) {
+        // re-encryption canceled.
+        LOG.info("Re-encryption was canceled.");
+        return;
+      }
       tracker.numFutureDone++;
       EncryptionFaultInjector.getInstance().reencryptUpdaterProcessOneTask();
       processTaskEntries(zonePath, task);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java

@@ -70,6 +70,14 @@ public class TopMetrics implements MetricsSource {
   public static final Logger LOG = LoggerFactory.getLogger(TopMetrics.class);
   public static final String TOPMETRICS_METRICS_SOURCE_NAME =
       "NNTopUserOpCounts";
+  /**
+   * In addition to counts of different RPC calls, NNTop also reports top
+   * users listing large directories (measured by the number of files involved
+   * in listing operations from the user). This is important because the CPU
+   * and GC overhead of a listing operation grows linearly with the number of
+   * files involved. This category in NNTop is {@link #FILES_IN_GETLISTING}.
+   */
+  public static final String FILES_IN_GETLISTING = "filesInGetListing";
   private final boolean isMetricsSourceEnabled;
 
   private static void logConf(Configuration conf) {
@@ -123,22 +131,30 @@ public class TopMetrics implements MetricsSource {
   public void report(boolean succeeded, String userName, InetAddress addr,
       String cmd, String src, String dst, FileStatus status) {
     // currently nntop only makes use of the username and the command
-    report(userName, cmd);
+    report(userName, cmd, 1);
   }
 
-  public void report(String userName, String cmd) {
+  public void reportFilesInGetListing(String userName, int numFiles) {
+    report(userName, FILES_IN_GETLISTING, numFiles);
+  }
+
+  public void report(String userName, String cmd, int delta) {
     long currTime = Time.monotonicNow();
-    report(currTime, userName, cmd);
+    report(currTime, userName, cmd, delta);
   }
 
-  public void report(long currTime, String userName, String cmd) {
+  public void report(long currTime, String userName, String cmd, int delta) {
     LOG.debug("a metric is reported: cmd: {} user: {}", cmd, userName);
     userName = UserGroupInformation.trimLoginMethod(userName);
     for (RollingWindowManager rollingWindowManager : rollingWindowManagers
         .values()) {
-      rollingWindowManager.recordMetric(currTime, cmd, userName, 1);
-      rollingWindowManager.recordMetric(currTime,
-          TopConf.ALL_CMDS, userName, 1);
+      rollingWindowManager.recordMetric(currTime, cmd, userName, delta);
+      // Increase the number of all RPC calls by the user, unless the report
+      // is for the number of files in a listing operation.
+      if (!cmd.equals(FILES_IN_GETLISTING)) {
+        rollingWindowManager.recordMetric(currTime,
+            TopConf.ALL_CMDS, userName, delta);
+      }
     }
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java

@@ -19,14 +19,13 @@
 package org.apache.hadoop.hdfs.server.protocol;
 
 import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
 import org.apache.hadoop.security.KerberosInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /** An inter-datanode protocol for updating generation stamp
  */
@@ -35,7 +34,7 @@ import org.apache.hadoop.security.KerberosInfo;
     clientPrincipal = DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY)
 @InterfaceAudience.Private
 public interface InterDatanodeProtocol {
-  public static final Log LOG = LogFactory.getLog(InterDatanodeProtocol.class);
+  Logger LOG = LoggerFactory.getLogger(InterDatanodeProtocol.class.getName());
 
   /**
    * Until version 9, this class InterDatanodeProtocol served as both

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java

@@ -753,7 +753,7 @@ public class DFSAdmin extends FsShell {
     } catch (SnapshotException e) {
       throw new RemoteException(e.getClass().getName(), e.getMessage());
     }
-    System.out.println("Allowing snaphot on " + argv[1] + " succeeded");
+    System.out.println("Allowing snapshot on " + argv[1] + " succeeded");
   }
   
   /**
@@ -770,7 +770,7 @@ public class DFSAdmin extends FsShell {
     } catch (SnapshotException e) {
       throw new RemoteException(e.getClass().getName(), e.getMessage());
     }
-    System.out.println("Disallowing snaphot on " + argv[1] + " succeeded");
+    System.out.println("Disallowing snapshot on " + argv[1] + " succeeded");
   }
   
   /**

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java

@@ -194,7 +194,8 @@ public class DFSZKFailoverController extends ZKFailoverController {
     try {
       retCode = zkfc.run(parser.getRemainingArgs());
     } catch (Throwable t) {
-      LOG.fatal("Got a fatal error, exiting now", t);
+      LOG.fatal("DFSZKFailOverController exiting due to earlier exception "
+          + t);
     }
     System.exit(retCode);
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto

@@ -46,6 +46,7 @@ message RequestInfoProto {
   // request itself, eg in the case that the node has
   // fallen behind.
   optional uint64 committedTxId = 4;
+  optional string nameServiceId = 5;
 }
 
 message SegmentStateProto {
@@ -73,6 +74,7 @@ message JournalRequestProto {
   required uint32 numTxns = 3;
   required bytes records = 4;
   required uint64 segmentTxnId = 5;
+  optional string nameServiceId = 6;
 }
 
 message JournalResponseProto { 
@@ -129,6 +131,7 @@ message PurgeLogsResponseProto {
  */
 message IsFormattedRequestProto {
   required JournalIdProto jid = 1;
+  optional string nameServiceId = 2;
 }
 
 message IsFormattedResponseProto {
@@ -140,6 +143,7 @@ message IsFormattedResponseProto {
  */
 message GetJournalCTimeRequestProto {
   required JournalIdProto jid = 1;
+  optional string nameServiceId = 2;
 }
 
 message GetJournalCTimeResponseProto {
@@ -172,6 +176,7 @@ message DoUpgradeResponseProto {
  */
 message DoFinalizeRequestProto {
   required JournalIdProto jid = 1;
+  optional string nameServiceId = 2;
 }
 
 message DoFinalizeResponseProto {
@@ -185,6 +190,7 @@ message CanRollBackRequestProto {
   required StorageInfoProto storage = 2;
   required StorageInfoProto prevStorage = 3;
   required int32 targetLayoutVersion = 4;
+  optional string nameServiceId = 5;
 }
 
 message CanRollBackResponseProto {
@@ -196,6 +202,7 @@ message CanRollBackResponseProto {
  */
 message DoRollbackRequestProto {
   required JournalIdProto jid = 1;
+  optional string nameserviceId = 2;
 }
 
 message DoRollbackResponseProto {
@@ -207,6 +214,7 @@ message DoRollbackResponseProto {
 message DiscardSegmentsRequestProto {
   required JournalIdProto jid = 1;
   required uint64 startTxId = 2;
+  optional string nameServiceId = 3;
 }
 
 message DiscardSegmentsResponseProto {
@@ -217,6 +225,7 @@ message DiscardSegmentsResponseProto {
  */
 message GetJournalStateRequestProto {
   required JournalIdProto jid = 1;
+  optional string nameServiceId = 2;
 }
 
 message GetJournalStateResponseProto {
@@ -232,6 +241,7 @@ message GetJournalStateResponseProto {
 message FormatRequestProto {
   required JournalIdProto jid = 1;
   required NamespaceInfoProto nsInfo = 2;
+  optional string nameServiceId = 3;
 }
 
 message FormatResponseProto {
@@ -244,6 +254,7 @@ message NewEpochRequestProto {
   required JournalIdProto jid = 1;
   required NamespaceInfoProto nsInfo = 2;
   required uint64 epoch = 3;
+  optional string nameServiceId = 4;
 }
 
 message NewEpochResponseProto {
@@ -259,6 +270,7 @@ message GetEditLogManifestRequestProto {
   // Whether or not the client will be reading from the returned streams.
   // optional bool forReading = 3 [default = true]; <obsolete, do not reuse>
   optional bool inProgressOk = 4 [default = false];
+  optional string nameServiceId = 5;
 }
 
 message GetEditLogManifestResponseProto {

hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml

@@ -4105,7 +4105,7 @@
 
 <property>
   <name>dfs.namenode.edits.asynclogging</name>
-  <value>false</value>
+  <value>true</value>
   <description>
     If set to true, enables asynchronous edit logs in the Namenode.  If set
     to false, the Namenode uses the traditional synchronous edit logs.

hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md

@@ -140,6 +140,7 @@ Deployment
 ### Enable Intel ISA-L
 
   HDFS native implementation of default RS codec leverages Intel ISA-L library to improve the encoding and decoding calculation. To enable and use Intel ISA-L, there are three steps.
+
   1. Build ISA-L library. Please refer to the official site "https://github.com/01org/isa-l/" for detail information.
   2. Build Hadoop with ISA-L support. Please refer to "Intel ISA-L build options" section in "Build instructions for Hadoop" in (BUILDING.txt) in the source code.
   3. Use `-Dbundle.isal` to copy the contents of the `isal.lib` directory into the final tar file. Deploy Hadoop with the tar file. Make sure ISA-L is available on HDFS clients and DataNodes.
@@ -192,7 +193,7 @@ Below are the details about each command.
 
  *  `[-addPolicies -policyFile <file>]`
 
-     Add a list of erasure coding policies. Please refer etc/hadoop/user_ec_policies.xml.template for the example policy file. The maximum cell size is defined in property 'dfs.namenode.ec.policies.max.cellsize' with the default value 4MB.
+     Add a list of erasure coding policies. Please refer etc/hadoop/user_ec_policies.xml.template for the example policy file. The maximum cell size is defined in property 'dfs.namenode.ec.policies.max.cellsize' with the default value 4MB. Currently HDFS allows the user to add 64 policies in total, and the added policy ID is in range of 64 to 127. Adding policy will fail if there are already 64 policies added.
 
  *  `[-listCodecs]`
 

hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md

@@ -91,7 +91,7 @@ In order to provide transparency with the old world, the ViewFs file system (i.e
 
 ViewFs implements the Hadoop file system interface just like HDFS and the local file system. It is a trivial file system in the sense that it only allows linking to other file systems. Because ViewFs implements the Hadoop file system interface, it works transparently Hadoop tools. For example, all the shell commands work with ViewFs as with HDFS and local file system.
 
-The mount points of a mount table are specified in the standard Hadoop configuration files. In the configuration of each cluster, the default file system is set to the mount table for that cluster as shown below (compare it with the configuration in [Single Namenode Clusters](#Single_Namenode_Clusters)).
+In the configuration of each cluster, the default file system is set to the mount table for that cluster as shown below (compare it with the configuration in [Single Namenode Clusters](#Single_Namenode_Clusters)).
 
 ```xml
 <property>
@@ -100,7 +100,47 @@ The mount points of a mount table are specified in the standard Hadoop configura
 </property>
 ```
 
-The authority following the `viewfs://` scheme in the URI is the mount table name. It is recommanded that the mount table of a cluster should be named by the cluster name. Then Hadoop system will look for a mount table with the name "clusterX" in the Hadoop configuration files. Operations arrange all gateways and service machines to contain the mount tables for ALL clusters such that, for each cluster, the default file system is set to the ViewFs mount table for that cluster as described above.
+The authority following the `viewfs://` scheme in the URI is the mount table name. It is recommended that the mount table of a cluster should be named by the cluster name. Then Hadoop system will look for a mount table with the name "clusterX" in the Hadoop configuration files. Operations arrange all gateways and service machines to contain the mount tables for ALL clusters such that, for each cluster, the default file system is set to the ViewFs mount table for that cluster as described above.
+
+The mount points of a mount table are specified in the standard Hadoop configuration files. All the mount table config entries for `viewfs` are prefixed by `fs.viewfs.mounttable.`. The mount points that are linking other filesystems are specified using `link` tags. The recommendation is to have mount points name same as in the linked filesystem target locations. For all namespaces that are not configured in the mount table, we can have them fallback to a default filesystem via `linkFallback`.
+
+In the below mount table configuration, namespace `/data` is linked to the filesystem `hdfs://nn1-clusterx.example.com:9820/data`, `/project` is linked to the filesystem `hdfs://nn2-clusterx.example.com:9820/project`. All namespaces that are not configured in the mount table, like `/logs` are linked to the filesystem `hdfs://nn5-clusterx.example.com:9820/home`.
+
+```xml
+<configuration>
+  <property>
+    <name>fs.viewfs.mounttable.ClusterX.link./data</name>
+    <value>hdfs://nn1-clusterx.example.com:9820/data</value>
+  </property>
+  <property>
+    <name>fs.viewfs.mounttable.ClusterX.link./project</name>
+    <value>hdfs://nn2-clusterx.example.com:9820/project</value>
+  </property>
+  <property>
+    <name>fs.viewfs.mounttable.ClusterX.link./user</name>
+    <value>hdfs://nn3-clusterx.example.com:9820/user</value>
+  </property>
+  <property>
+    <name>fs.viewfs.mounttable.ClusterX.link./tmp</name>
+    <value>hdfs://nn4-clusterx.example.com:9820/tmp</value>
+  </property>
+  <property>
+    <name>fs.viewfs.mounttable.ClusterX.linkFallback</name>
+    <value>hdfs://nn5-clusterx.example.com:9820/home</value>
+  </property>
+</configuration>
+```
+
+Alternatively we can have the mount table's root merged with the root of another filesystem via `linkMergeSlash`. In the below mount table configuration, ClusterY's root is merged with the root filesystem at `hdfs://nn1-clustery.example.com:9820`.
+
+```xml
+<configuration>
+  <property>
+    <name>fs.viewfs.mounttable.ClusterY.linkMergeSlash</name>
+    <value>hdfs://nn1-clustery.example.com:9820/</value>
+  </property>
+</configuration>
+```
 
 ### Pathname Usage Patterns
 

hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md

@@ -467,6 +467,8 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileSt
               {
                 "accessTime"      : 1320171722771,
                 "blockSize"       : 33554432,
+                "childrenNum"     : 0,
+                "fileId"          : 16388,
                 "group"           : "supergroup",
                 "length"          : 24930,
                 "modificationTime": 1320171722771,
@@ -474,11 +476,14 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileSt
                 "pathSuffix"      : "a.patch",
                 "permission"      : "644",
                 "replication"     : 1,
+                "storagePolicy"   : 0,
                 "type"            : "FILE"
               },
               {
                 "accessTime"      : 0,
                 "blockSize"       : 0,
+                "childrenNum"     : 0,
+                "fileId"          : 16389,
                 "group"           : "supergroup",
                 "length"          : 0,
                 "modificationTime": 1320895981256,

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLinkFallback.java

@@ -0,0 +1,264 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.viewfs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import javax.security.auth.login.LoginException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystemTestHelper;
+import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test for viewfs with LinkFallback mount table entries.
+ */
+public class TestViewFileSystemLinkFallback extends ViewFileSystemBaseTest {
+
+  private static FileSystem fsDefault;
+  private static MiniDFSCluster cluster;
+  private static final int NAME_SPACES_COUNT = 3;
+  private static final int DATA_NODES_COUNT = 3;
+  private static final int FS_INDEX_DEFAULT = 0;
+  private static final String LINK_FALLBACK_CLUSTER_1_NAME = "Cluster1";
+  private static final FileSystem[] FS_HDFS = new FileSystem[NAME_SPACES_COUNT];
+  private static final Configuration CONF = new Configuration();
+  private static final File TEST_DIR = GenericTestUtils.getTestDir(
+      TestViewFileSystemLinkFallback.class.getSimpleName());
+  private static final String TEST_BASE_PATH =
+      "/tmp/TestViewFileSystemLinkFallback";
+  private final static Logger LOG = LoggerFactory.getLogger(
+      TestViewFileSystemLinkFallback.class);
+
+
+  @Override
+  protected FileSystemTestHelper createFileSystemHelper() {
+    return new FileSystemTestHelper(TEST_BASE_PATH);
+  }
+
+  @BeforeClass
+  public static void clusterSetupAtBeginning() throws IOException,
+      LoginException, URISyntaxException {
+    SupportsBlocks = true;
+    CONF.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY,
+        true);
+    cluster = new MiniDFSCluster.Builder(CONF)
+        .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(
+            NAME_SPACES_COUNT))
+        .numDataNodes(DATA_NODES_COUNT)
+        .build();
+    cluster.waitClusterUp();
+
+    for (int i = 0; i < NAME_SPACES_COUNT; i++) {
+      FS_HDFS[i] = cluster.getFileSystem(i);
+    }
+    fsDefault = FS_HDFS[FS_INDEX_DEFAULT];
+  }
+
+  @AfterClass
+  public static void clusterShutdownAtEnd() throws Exception {
+    if (cluster != null) {
+      cluster.shutdown();
+    }
+  }
+
+  @Override
+  @Before
+  public void setUp() throws Exception {
+    fsTarget = fsDefault;
+    super.setUp();
+  }
+
+  /**
+   * Override this so that we don't set the targetTestRoot to any path under the
+   * root of the FS, and so that we don't try to delete the test dir, but rather
+   * only its contents.
+   */
+  @Override
+  void initializeTargetTestRoot() throws IOException {
+    targetTestRoot = fsDefault.makeQualified(new Path("/"));
+    for (FileStatus status : fsDefault.listStatus(targetTestRoot)) {
+      fsDefault.delete(status.getPath(), true);
+    }
+  }
+
+  @Override
+  void setupMountPoints() {
+    super.setupMountPoints();
+    ConfigUtil.addLinkFallback(conf, LINK_FALLBACK_CLUSTER_1_NAME,
+        targetTestRoot.toUri());
+  }
+
+  @Override
+  int getExpectedDelegationTokenCount() {
+    return 1; // all point to the same fs so 1 unique token
+  }
+
+  @Override
+  int getExpectedDelegationTokenCountWithCredentials() {
+    return 1;
+  }
+
+  @Test
+  public void testConfLinkFallback() throws Exception {
+    Path testBasePath = new Path(TEST_BASE_PATH);
+    Path testLevel2Dir = new Path(TEST_BASE_PATH, "dir1/dirA");
+    Path testBaseFile = new Path(testBasePath, "testBaseFile.log");
+    Path testBaseFileRelative = new Path(testLevel2Dir,
+        "../../testBaseFile.log");
+    Path testLevel2File = new Path(testLevel2Dir, "testLevel2File.log");
+    fsTarget.mkdirs(testLevel2Dir);
+
+    fsTarget.createNewFile(testBaseFile);
+    FSDataOutputStream dataOutputStream = fsTarget.append(testBaseFile);
+    dataOutputStream.write(1);
+    dataOutputStream.close();
+
+    fsTarget.createNewFile(testLevel2File);
+    dataOutputStream = fsTarget.append(testLevel2File);
+    dataOutputStream.write("test link fallback".toString().getBytes());
+    dataOutputStream.close();
+
+    String clusterName = "ClusterFallback";
+    URI viewFsUri = new URI(FsConstants.VIEWFS_SCHEME, clusterName,
+        "/", null, null);
+
+    Configuration conf = new Configuration();
+    ConfigUtil.addLinkFallback(conf, clusterName, fsTarget.getUri());
+
+    FileSystem vfs = FileSystem.get(viewFsUri, conf);
+    assertEquals(ViewFileSystem.class, vfs.getClass());
+    FileStatus baseFileStat = vfs.getFileStatus(new Path(viewFsUri.toString()
+        + testBaseFile.toUri().toString()));
+    LOG.info("BaseFileStat: " + baseFileStat);
+    FileStatus baseFileRelStat = vfs.getFileStatus(new Path(viewFsUri.toString()
+        + testBaseFileRelative.toUri().toString()));
+    LOG.info("BaseFileRelStat: " + baseFileRelStat);
+    Assert.assertEquals("Unexpected file length for " + testBaseFile,
+        1, baseFileStat.getLen());
+    Assert.assertEquals("Unexpected file length for " + testBaseFileRelative,
+        baseFileStat.getLen(), baseFileRelStat.getLen());
+    FileStatus level2FileStat = vfs.getFileStatus(new Path(viewFsUri.toString()
+        + testLevel2File.toUri().toString()));
+    LOG.info("Level2FileStat: " + level2FileStat);
+    vfs.close();
+  }
+
+  @Test
+  public void testConfLinkFallbackWithRegularLinks() throws Exception {
+    Path testBasePath = new Path(TEST_BASE_PATH);
+    Path testLevel2Dir = new Path(TEST_BASE_PATH, "dir1/dirA");
+    Path testBaseFile = new Path(testBasePath, "testBaseFile.log");
+    Path testLevel2File = new Path(testLevel2Dir, "testLevel2File.log");
+    fsTarget.mkdirs(testLevel2Dir);
+
+    fsTarget.createNewFile(testBaseFile);
+    fsTarget.createNewFile(testLevel2File);
+    FSDataOutputStream dataOutputStream = fsTarget.append(testLevel2File);
+    dataOutputStream.write("test link fallback".toString().getBytes());
+    dataOutputStream.close();
+
+    String clusterName = "ClusterFallback";
+    URI viewFsUri = new URI(FsConstants.VIEWFS_SCHEME, clusterName,
+        "/", null, null);
+
+    Configuration conf = new Configuration();
+    ConfigUtil.addLink(conf, clusterName,
+        "/internalDir/linkToDir2",
+        new Path(targetTestRoot, "dir2").toUri());
+    ConfigUtil.addLink(conf, clusterName,
+        "/internalDir/internalDirB/linkToDir3",
+        new Path(targetTestRoot, "dir3").toUri());
+    ConfigUtil.addLink(conf, clusterName,
+        "/danglingLink",
+        new Path(targetTestRoot, "missingTarget").toUri());
+    ConfigUtil.addLink(conf, clusterName,
+        "/linkToAFile",
+        new Path(targetTestRoot, "aFile").toUri());
+    System.out.println("ViewFs link fallback " + fsTarget.getUri());
+    ConfigUtil.addLinkFallback(conf, clusterName, targetTestRoot.toUri());
+
+    FileSystem vfs = FileSystem.get(viewFsUri, conf);
+    assertEquals(ViewFileSystem.class, vfs.getClass());
+    FileStatus baseFileStat = vfs.getFileStatus(
+        new Path(viewFsUri.toString() + testBaseFile.toUri().toString()));
+    LOG.info("BaseFileStat: " + baseFileStat);
+    Assert.assertEquals("Unexpected file length for " + testBaseFile,
+        0, baseFileStat.getLen());
+    FileStatus level2FileStat = vfs.getFileStatus(new Path(viewFsUri.toString()
+        + testLevel2File.toUri().toString()));
+    LOG.info("Level2FileStat: " + level2FileStat);
+
+    dataOutputStream = vfs.append(testLevel2File);
+    dataOutputStream.write("Writing via viewfs fallback path".getBytes());
+    dataOutputStream.close();
+
+    FileStatus level2FileStatAfterWrite = vfs.getFileStatus(
+        new Path(viewFsUri.toString() + testLevel2File.toUri().toString()));
+    Assert.assertTrue("Unexpected file length for " + testLevel2File,
+        level2FileStatAfterWrite.getLen() > level2FileStat.getLen());
+
+    vfs.close();
+  }
+
+  @Test
+  public void testConfLinkFallbackWithMountPoint() throws Exception {
+    TEST_DIR.mkdirs();
+    Configuration conf = new Configuration();
+    String clusterName = "ClusterX";
+    String mountPoint = "/user";
+    URI viewFsUri = new URI(FsConstants.VIEWFS_SCHEME, clusterName,
+        "/", null, null);
+    String expectedErrorMsg =  "Invalid linkFallback entry in config: " +
+        "linkFallback./user";
+    String mountTableEntry = Constants.CONFIG_VIEWFS_PREFIX + "."
+        + clusterName + "." + Constants.CONFIG_VIEWFS_LINK_FALLBACK
+        + "." + mountPoint;
+    conf.set(mountTableEntry, TEST_DIR.toURI().toString());
+
+    try {
+      FileSystem.get(viewFsUri, conf);
+      fail("Shouldn't allow linkMergeSlash to take extra mount points!");
+    } catch (IOException e) {
+      assertTrue("Unexpected error: " + e.getMessage(),
+          e.getMessage().contains(expectedErrorMsg));
+    }
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLinkMergeSlash.java

@@ -0,0 +1,234 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.viewfs;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystemTestHelper;
+import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import javax.security.auth.login.LoginException;
+
+/**
+ * Test for viewfs with LinkMergeSlash mount table entries.
+ */
+public class TestViewFileSystemLinkMergeSlash extends ViewFileSystemBaseTest {
+
+  private static FileSystem fsDefault;
+  private static MiniDFSCluster cluster;
+  private static final int NAME_SPACES_COUNT = 3;
+  private static final int DATA_NODES_COUNT = 3;
+  private static final int FS_INDEX_DEFAULT = 0;
+  private static final String LINK_MERGE_SLASH_CLUSTER_1_NAME = "ClusterLMS1";
+  private static final String LINK_MERGE_SLASH_CLUSTER_2_NAME = "ClusterLMS2";
+  private static final FileSystem[] FS_HDFS = new FileSystem[NAME_SPACES_COUNT];
+  private static final Configuration CONF = new Configuration();
+  private static final File TEST_DIR = GenericTestUtils.getTestDir(
+      TestViewFileSystemLinkMergeSlash.class.getSimpleName());
+  private static final String TEST_TEMP_PATH =
+      "/tmp/TestViewFileSystemLinkMergeSlash";
+  private final static Logger LOG = LoggerFactory.getLogger(
+      TestViewFileSystemLinkMergeSlash.class);
+
+  @Override
+  protected FileSystemTestHelper createFileSystemHelper() {
+    return new FileSystemTestHelper(TEST_TEMP_PATH);
+  }
+
+  @BeforeClass
+  public static void clusterSetupAtBeginning() throws IOException,
+      LoginException, URISyntaxException {
+    SupportsBlocks = true;
+    CONF.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY,
+        true);
+    cluster = new MiniDFSCluster.Builder(CONF)
+        .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(
+            NAME_SPACES_COUNT))
+        .numDataNodes(DATA_NODES_COUNT)
+        .build();
+    cluster.waitClusterUp();
+
+    for (int i = 0; i < NAME_SPACES_COUNT; i++) {
+      FS_HDFS[i] = cluster.getFileSystem(i);
+    }
+    fsDefault = FS_HDFS[FS_INDEX_DEFAULT];
+  }
+
+  @AfterClass
+  public static void clusterShutdownAtEnd() throws Exception {
+    if (cluster != null) {
+      cluster.shutdown();
+    }
+  }
+
+  @Override
+  @Before
+  public void setUp() throws Exception {
+    fsTarget = fsDefault;
+    super.setUp();
+  }
+
+  /**
+   * Override this so that we don't set the targetTestRoot to any path under the
+   * root of the FS, and so that we don't try to delete the test dir, but rather
+   * only its contents.
+   */
+  @Override
+  void initializeTargetTestRoot() throws IOException {
+    targetTestRoot = fsDefault.makeQualified(new Path("/"));
+    for (FileStatus status : fsDefault.listStatus(targetTestRoot)) {
+      fsDefault.delete(status.getPath(), true);
+    }
+  }
+
+  @Override
+  void setupMountPoints() {
+    super.setupMountPoints();
+    ConfigUtil.addLinkMergeSlash(conf, LINK_MERGE_SLASH_CLUSTER_1_NAME,
+        targetTestRoot.toUri());
+    ConfigUtil.addLinkMergeSlash(conf, LINK_MERGE_SLASH_CLUSTER_2_NAME,
+        targetTestRoot.toUri());
+  }
+
+  @Override
+  int getExpectedDelegationTokenCount() {
+    return 1; // all point to the same fs so 1 unique token
+  }
+
+  @Override
+  int getExpectedDelegationTokenCountWithCredentials() {
+    return 1;
+  }
+
+  @Test
+  public void testConfLinkMergeSlash() throws Exception {
+    TEST_DIR.mkdirs();
+    String clusterName = "ClusterMerge";
+    URI viewFsUri = new URI(FsConstants.VIEWFS_SCHEME, clusterName,
+        "/", null, null);
+    String testFileName = "testLinkMergeSlash";
+
+    File infile = new File(TEST_DIR, testFileName);
+    final byte[] content = "HelloWorld".getBytes();
+    FileOutputStream fos = null;
+    try {
+      fos = new FileOutputStream(infile);
+      fos.write(content);
+    } finally {
+      if (fos != null) {
+        fos.close();
+      }
+    }
+    assertEquals((long)content.length, infile.length());
+
+    Configuration conf = new Configuration();
+    ConfigUtil.addLinkMergeSlash(conf, clusterName, TEST_DIR.toURI());
+
+    FileSystem vfs = FileSystem.get(viewFsUri, conf);
+    assertEquals(ViewFileSystem.class, vfs.getClass());
+    FileStatus stat = vfs.getFileStatus(new Path(viewFsUri.toString() +
+        testFileName));
+
+    LOG.info("File stat: " + stat);
+    vfs.close();
+  }
+
+  @Test
+  public void testConfLinkMergeSlashWithRegularLinks() throws Exception {
+    TEST_DIR.mkdirs();
+    String clusterName = "ClusterMerge";
+    String expectedErrorMsg1 = "Mount table ClusterMerge has already been " +
+        "configured with a merge slash link";
+    String expectedErrorMsg2 = "Mount table ClusterMerge has already been " +
+        "configured with regular links";
+    URI viewFsUri = new URI(FsConstants.VIEWFS_SCHEME, clusterName,
+        "/", null, null);
+    Configuration conf = new Configuration();
+    ConfigUtil.addLinkMergeSlash(conf, clusterName, TEST_DIR.toURI());
+    ConfigUtil.addLink(conf, clusterName, "testDir", TEST_DIR.toURI());
+
+    try {
+      FileSystem.get(viewFsUri, conf);
+      fail("Shouldn't allow both merge slash link and regular link on same "
+          + "mount table.");
+    } catch (IOException e) {
+      assertTrue("Unexpected error message: " + e.getMessage(),
+          e.getMessage().contains(expectedErrorMsg1) || e.getMessage()
+              .contains(expectedErrorMsg2));
+    }
+  }
+
+  @Test
+  public void testConfLinkMergeSlashWithMountPoint() throws Exception {
+    TEST_DIR.mkdirs();
+    Configuration conf = new Configuration();
+    String clusterName = "ClusterX";
+    String mountPoint = "/user";
+    URI viewFsUri = new URI(FsConstants.VIEWFS_SCHEME, clusterName,
+        "/", null, null);
+    String expectedErrorMsg =  "Invalid linkMergeSlash entry in config: " +
+        "linkMergeSlash./user";
+    String mountTableEntry = Constants.CONFIG_VIEWFS_PREFIX + "."
+        + clusterName + "." + Constants.CONFIG_VIEWFS_LINK_MERGE_SLASH
+        + "." + mountPoint;
+    conf.set(mountTableEntry, TEST_DIR.toURI().toString());
+
+    try {
+      FileSystem.get(viewFsUri, conf);
+      fail("Shouldn't allow linkMergeSlash to take extra mount points!");
+    } catch (IOException e) {
+      assertTrue(e.getMessage().contains(expectedErrorMsg));
+    }
+  }
+
+  @Test
+  public void testChildFileSystems() throws Exception {
+    URI viewFsUri = new URI(FsConstants.VIEWFS_SCHEME,
+        LINK_MERGE_SLASH_CLUSTER_1_NAME, "/", null, null);
+    FileSystem fs = FileSystem.get(viewFsUri, conf);
+    FileSystem[] childFs = fs.getChildFileSystems();
+    Assert.assertEquals("Unexpected number of child filesystems!",
+        1, childFs.length);
+    Assert.assertEquals("Unexpected child filesystem!",
+        DistributedFileSystem.class, childFs[0].getClass());
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java

@@ -502,7 +502,11 @@ public class StripedFileTestUtil {
         dataBytes.length, parityBytes.length);
     final RawErasureEncoder encoder =
         CodecUtil.createRawEncoder(conf, codecName, coderOptions);
-    encoder.encode(dataBytes, expectedParityBytes);
+    try {
+      encoder.encode(dataBytes, expectedParityBytes);
+    } catch (IOException e) {
+      Assert.fail("Unexpected IOException: " + e.getMessage());
+    }
     for (int i = 0; i < parityBytes.length; i++) {
       if (checkSet.contains(i + dataBytes.length)){
         Assert.assertArrayEquals("i=" + i, expectedParityBytes[i],

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java

@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
@@ -38,6 +39,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ECSchema;
 import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
 import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
 import org.apache.hadoop.security.token.Token;
@@ -77,30 +79,29 @@ public class TestDFSStripedOutputStreamWithFailure {
         .getLogger().setLevel(Level.ALL);
   }
 
+  private final int cellSize = 64 * 1024; //64k
+  private final int stripesPerBlock = 4;
   private ErasureCodingPolicy ecPolicy;
   private int dataBlocks;
   private int parityBlocks;
-  private int cellSize;
-  private final int stripesPerBlock = 4;
   private int blockSize;
   private int blockGroupSize;
 
   private static final int FLUSH_POS =
       9 * DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT + 1;
 
-  public ErasureCodingPolicy getEcPolicy() {
-    return StripedFileTestUtil.getDefaultECPolicy();
+  public ECSchema getEcSchema() {
+    return StripedFileTestUtil.getDefaultECPolicy().getSchema();
   }
 
   /*
    * Initialize erasure coding policy.
    */
   @Before
-  public void init(){
-    ecPolicy = getEcPolicy();
+  public void init() {
+    ecPolicy = new ErasureCodingPolicy(getEcSchema(), cellSize);
     dataBlocks = ecPolicy.getNumDataUnits();
     parityBlocks = ecPolicy.getNumParityUnits();
-    cellSize = ecPolicy.getCellSize();
     blockSize = cellSize * stripesPerBlock;
     blockGroupSize = blockSize * dataBlocks;
     dnIndexSuite = getDnIndexSuite();
@@ -189,7 +190,7 @@ public class TestDFSStripedOutputStreamWithFailure {
   private List<Integer> lengths;
 
   Integer getLength(int i) {
-    return i >= 0 && i < lengths.size()? lengths.get(i): null;
+    return i >= 0 && i < lengths.size() ? lengths.get(i): null;
   }
 
   private static final Random RANDOM = new Random();
@@ -220,6 +221,10 @@ public class TestDFSStripedOutputStreamWithFailure {
     cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
     cluster.waitActive();
     dfs = cluster.getFileSystem();
+    AddErasureCodingPolicyResponse[] res =
+        dfs.addErasureCodingPolicies(new ErasureCodingPolicy[]{ecPolicy});
+    ecPolicy = res[0].getPolicy();
+    dfs.enableErasureCodingPolicy(ecPolicy.getName());
     DFSTestUtil.enableAllECPolicies(dfs);
     dfs.mkdirs(dir);
     dfs.setErasureCodingPolicy(dir, ecPolicy.getName());
@@ -241,7 +246,7 @@ public class TestDFSStripedOutputStreamWithFailure {
     return conf;
   }
 
-  @Test(timeout=240000)
+  @Test(timeout=300000)
   public void testMultipleDatanodeFailure56() throws Exception {
     runTestWithMultipleFailure(getLength(56));
   }
@@ -260,7 +265,8 @@ public class TestDFSStripedOutputStreamWithFailure {
 
   @Test(timeout=240000)
   public void testBlockTokenExpired() throws Exception {
-    final int length = dataBlocks * (blockSize - cellSize);
+    // Make sure killPos is greater than the length of one stripe
+    final int length = dataBlocks * cellSize * 3;
     final HdfsConfiguration conf = newHdfsConfiguration();
 
     conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
@@ -300,13 +306,13 @@ public class TestDFSStripedOutputStreamWithFailure {
       cluster.triggerHeartbeats();
       DatanodeInfo[] info = dfs.getClient().datanodeReport(
           DatanodeReportType.LIVE);
-      assertEquals("Mismatches number of live Dns ", numDatanodes, info.length);
+      assertEquals("Mismatches number of live Dns", numDatanodes, info.length);
       final Path dirFile = new Path(dir, "ecfile");
       LambdaTestUtils.intercept(
           IOException.class,
           "File " + dirFile + " could only be written to " +
               numDatanodes + " of the " + dataBlocks + " required nodes for " +
-              getEcPolicy().getName(),
+              ecPolicy.getName(),
           () -> {
             try (FSDataOutputStream out = dfs.create(dirFile, true)) {
               out.write("something".getBytes());
@@ -319,6 +325,82 @@ public class TestDFSStripedOutputStreamWithFailure {
     }
   }
 
+  private void testCloseWithExceptionsInStreamer(
+      int numFailures, boolean shouldFail) throws Exception {
+    assertTrue(numFailures <=
+        ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits());
+    final Path dirFile = new Path(dir, "ecfile-" + numFailures);
+    try (FSDataOutputStream out = dfs.create(dirFile, true)) {
+      out.write("idempotent close".getBytes());
+
+      // Expect to raise IOE on the first close call, but any following
+      // close() should be no-op.
+      LambdaTestUtils.intercept(IOException.class,
+          out::close);
+
+      assertTrue(out.getWrappedStream() instanceof DFSStripedOutputStream);
+      DFSStripedOutputStream stripedOut =
+          (DFSStripedOutputStream) out.getWrappedStream();
+      for (int i = 0; i < numFailures; i++) {
+        // Only inject 1 stream failure.
+        stripedOut.getStripedDataStreamer(i).getLastException().set(
+            new IOException("injected failure")
+        );
+      }
+      if (shouldFail) {
+        LambdaTestUtils.intercept(IOException.class, out::close);
+      }
+
+      // Close multiple times. All the following close() should have no
+      // side-effect.
+      out.close();
+    }
+  }
+
+  // HDFS-12612
+  @Test
+  public void testIdempotentCloseWithFailedStreams() throws Exception {
+    HdfsConfiguration conf = new HdfsConfiguration();
+    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+    try {
+      setup(conf);
+      // shutdown few datanodes to avoid getting sufficient data blocks number
+      // of datanodes.
+      while (cluster.getDataNodes().size() >= dataBlocks) {
+        cluster.stopDataNode(0);
+      }
+      cluster.restartNameNodes();
+      cluster.triggerHeartbeats();
+
+      testCloseWithExceptionsInStreamer(1, false);
+      testCloseWithExceptionsInStreamer(ecPolicy.getNumParityUnits(), false);
+      testCloseWithExceptionsInStreamer(ecPolicy.getNumParityUnits() + 1, true);
+      testCloseWithExceptionsInStreamer(ecPolicy.getNumDataUnits(), true);
+    } finally {
+      tearDown();
+    }
+  }
+
+  @Test
+  public void testCloseAfterAbort() throws Exception {
+    HdfsConfiguration conf = new HdfsConfiguration();
+    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+    try {
+      setup(conf);
+
+      final Path dirFile = new Path(dir, "ecfile");
+      FSDataOutputStream out = dfs.create(dirFile, true);
+      assertTrue(out.getWrappedStream() instanceof DFSStripedOutputStream);
+      DFSStripedOutputStream stripedOut =
+          (DFSStripedOutputStream) out.getWrappedStream();
+      stripedOut.abort();
+      LambdaTestUtils.intercept(IOException.class,
+          "Lease timeout", stripedOut::close);
+    } finally {
+      tearDown();
+    }
+  }
+
   @Test(timeout = 90000)
   public void testAddBlockWhenNoSufficientParityNumOfNodes()
       throws IOException {
@@ -337,7 +419,7 @@ public class TestDFSStripedOutputStreamWithFailure {
       cluster.triggerHeartbeats();
       DatanodeInfo[] info = dfs.getClient().datanodeReport(
           DatanodeReportType.LIVE);
-      assertEquals("Mismatches number of live Dns ", numDatanodes, info.length);
+      assertEquals("Mismatches number of live Dns", numDatanodes, info.length);
       Path srcPath = new Path(dir, "testAddBlockWhenNoSufficientParityNodes");
       int fileLength = cellSize - 1000;
       final byte[] expected = StripedFileTestUtil.generateBytes(fileLength);
@@ -356,7 +438,7 @@ public class TestDFSStripedOutputStreamWithFailure {
       try {
         LOG.info("runTest: dn=" + dn + ", length=" + length);
         setup(conf);
-        runTest(length, new int[]{length/2}, new int[]{dn}, false);
+        runTest(length, new int[]{length / 2}, new int[]{dn}, false);
       } catch (Throwable e) {
         final String err = "failed, dn=" + dn + ", length=" + length
             + StringUtils.stringifyException(e);
@@ -506,10 +588,10 @@ public class TestDFSStripedOutputStreamWithFailure {
     long oldGS = -1; // the old GS before bumping
     List<Long> gsList = new ArrayList<>();
     final List<DatanodeInfo> killedDN = new ArrayList<>();
-    int numKilled=0;
+    int numKilled = 0;
     for(; pos.get() < length;) {
       final int i = pos.getAndIncrement();
-      if (numKilled < killPos.length &&  i == killPos[numKilled]) {
+      if (numKilled < killPos.length && i == killPos[numKilled]) {
         assertTrue(firstGS != -1);
         final long gs = getGenerationStamp(stripedOut);
         if (numKilled == 0) {
@@ -630,8 +712,6 @@ public class TestDFSStripedOutputStreamWithFailure {
 
   private void run(int offset) {
     int base = getBase();
-    // TODO: Fix and re-enable these flaky tests. See HDFS-12417.
-    assumeTrue("Test has been temporarily disabled. See HDFS-12417.", false);
     assumeTrue(base >= 0);
     final int i = offset + base;
     final Integer length = getLength(i);

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureWithRandomECPolicy.java

@@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.io.erasurecode.ECSchema;
 
 /**
  * This tests write operation of DFS striped file with a random erasure code
@@ -28,18 +28,18 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 public class TestDFSStripedOutputStreamWithFailureWithRandomECPolicy extends
     TestDFSStripedOutputStreamWithFailure {
 
+  private final ECSchema schema;
+
   private static final Log LOG = LogFactory.getLog(
       TestDFSStripedOutputStreamWithRandomECPolicy.class.getName());
 
-  private ErasureCodingPolicy ecPolicy;
-
   public TestDFSStripedOutputStreamWithFailureWithRandomECPolicy() {
-    ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy();
-    LOG.info(ecPolicy);
+    schema = StripedFileTestUtil.getRandomNonDefaultECPolicy().getSchema();
+    LOG.info(schema);
   }
 
   @Override
-  public ErasureCodingPolicy getEcPolicy() {
-    return ecPolicy;
+  public ECSchema getEcSchema() {
+    return schema;
   }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java

@@ -45,7 +45,8 @@ public class TestDatanodeDeath {
     DFSTestUtil.setNameNodeLogLevel(Level.ALL);
     GenericTestUtils.setLogLevel(DataNode.LOG, Level.ALL);
     GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL);
-    GenericTestUtils.setLogLevel(InterDatanodeProtocol.LOG, Level.ALL);
+    GenericTestUtils.setLogLevel(InterDatanodeProtocol.LOG, org.slf4j
+        .event.Level.TRACE);
   }
 
   static final int blockSize = 8192;