HDFS-9835. OIV: add ReverseXML processor which reconstructs an fsimage from an XML file (cmccabe)

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java

@@ -17,8 +17,12 @@
  */
 package org.apache.hadoop.test;
 
+import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.StringWriter;
 import java.lang.management.ManagementFactory;
 import java.lang.management.ThreadInfo;
@@ -32,6 +36,7 @@ import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Pattern;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.fs.FileUtil;
@@ -481,4 +486,55 @@ public abstract class GenericTestUtils {
     Assume.assumeTrue(
         Boolean.parseBoolean(System.getProperty("runningWithNative", "false")));
   }
+
+  /**
+   * Get the diff between two files.
+   *
+   * @param a
+   * @param b
+   * @return The empty string if there is no diff; the diff, otherwise.
+   *
+   * @throws IOException If there is an error reading either file.
+   */
+  public static String getFilesDiff(File a, File b) throws IOException {
+    StringBuilder bld = new StringBuilder();
+    BufferedReader ra = null, rb = null;
+    try {
+      ra = new BufferedReader(
+          new InputStreamReader(new FileInputStream(a)));
+      rb = new BufferedReader(
+          new InputStreamReader(new FileInputStream(b)));
+      while (true) {
+        String la = ra.readLine();
+        String lb = rb.readLine();
+        if (la == null) {
+          if (lb != null) {
+            addPlusses(bld, ra);
+          }
+          break;
+        } else if (lb == null) {
+          if (la != null) {
+            addPlusses(bld, rb);
+          }
+          break;
+        }
+        if (!la.equals(lb)) {
+          bld.append(" - ").append(la).append("\n");
+          bld.append(" + ").append(lb).append("\n");
+        }
+      }
+    } finally {
+      IOUtils.closeQuietly(ra);
+      IOUtils.closeQuietly(rb);
+    }
+    return bld.toString();
+  }
+
+  private static void addPlusses(StringBuilder bld, BufferedReader r)
+      throws IOException {
+    String l;
+    while ((l = r.readLine()) != null) {
+      bld.append(" + ").append(l).append("\n");
+    }
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -980,6 +980,8 @@ Release 2.9.0 - UNRELEASED
     HDFS-9047. Retire libwebhdfs. (wheat9)
 
   NEW FEATURES
+    HDFS-9835. OIV: add ReverseXML processor which reconstructs an fsimage from
+    an XML file (cmccabe)
 
   IMPROVEMENTS
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java

@@ -90,14 +90,14 @@ public final class FSImageFormatPBINode {
   private static final AclEntryType[] ACL_ENTRY_TYPE_VALUES = AclEntryType
       .values();
   
-  private static final int XATTR_NAMESPACE_MASK = 3;
-  private static final int XATTR_NAMESPACE_OFFSET = 30;
-  private static final int XATTR_NAME_MASK = (1 << 24) - 1;
-  private static final int XATTR_NAME_OFFSET = 6;
+  public static final int XATTR_NAMESPACE_MASK = 3;
+  public static final int XATTR_NAMESPACE_OFFSET = 30;
+  public static final int XATTR_NAME_MASK = (1 << 24) - 1;
+  public static final int XATTR_NAME_OFFSET = 6;
 
   /* See the comments in fsimage.proto for an explanation of the following. */
-  private static final int XATTR_NAMESPACE_EXT_OFFSET = 5;
-  private static final int XATTR_NAMESPACE_EXT_MASK = 1;
+  public static final int XATTR_NAMESPACE_EXT_OFFSET = 5;
+  public static final int XATTR_NAMESPACE_EXT_MASK = 1;
 
   private static final XAttr.NameSpace[] XATTR_NAMESPACE_VALUES =
       XAttr.NameSpace.values();

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java

@@ -0,0 +1,1639 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_MASK;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAME_OFFSET;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_OFFSET;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_EXT_OFFSET;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_EXT_MASK;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.Map;
+
+import com.google.common.io.CountingOutputStream;
+import com.google.common.primitives.Ints;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.TextFormat;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoExpirationProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos;
+import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName;
+import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.CacheManagerSection;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.NameSystemSection;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotDiffSection.DiffEntry;
+import org.apache.hadoop.hdfs.util.MD5FileUtils;
+import org.apache.hadoop.hdfs.util.XMLUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.MD5Hash;
+import org.apache.hadoop.util.StringUtils;
+
+import javax.xml.bind.annotation.adapters.HexBinaryAdapter;
+import javax.xml.stream.XMLEventReader;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.events.XMLEvent;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class OfflineImageReconstructor {
+  public static final Log LOG =
+      LogFactory.getLog(OfflineImageReconstructor.class);
+
+  /**
+   * The output stream.
+   */
+  private final CountingOutputStream out;
+
+  /**
+   * A source of XML events based on the input file.
+   */
+  private final XMLEventReader events;
+
+  /**
+   * A map of section names to section handler objects.
+   */
+  private final HashMap<String, SectionProcessor> sections;
+
+  /**
+   * The offset of the start of the current section.
+   */
+  private long sectionStartOffset;
+
+  /**
+   * The FileSummary builder, where we gather information about each section
+   * we wrote.
+   */
+  private final FileSummary.Builder fileSummaryBld =
+      FileSummary.newBuilder();
+
+  /**
+   * The string table.  See registerStringId for details.
+   */
+  private final HashMap<String, Integer> stringTable = new HashMap<>();
+
+  /**
+   * The date formatter to use with fsimage XML files.
+   */
+  private final SimpleDateFormat isoDateFormat;
+
+  /**
+   * The latest string ID.  See registerStringId for details.
+   */
+  private int latestStringId = 0;
+
+  private OfflineImageReconstructor(CountingOutputStream out,
+      InputStreamReader reader) throws XMLStreamException {
+    this.out = out;
+    XMLInputFactory factory = XMLInputFactory.newInstance();
+    this.events = factory.createXMLEventReader(reader);
+    this.sections = new HashMap<>();
+    this.sections.put(NameSectionProcessor.NAME, new NameSectionProcessor());
+    this.sections.put(INodeSectionProcessor.NAME, new INodeSectionProcessor());
+    this.sections.put(SecretManagerSectionProcessor.NAME,
+        new SecretManagerSectionProcessor());
+    this.sections.put(CacheManagerSectionProcessor.NAME,
+        new CacheManagerSectionProcessor());
+    this.sections.put(SnapshotDiffSectionProcessor.NAME,
+        new SnapshotDiffSectionProcessor());
+    this.sections.put(INodeReferenceSectionProcessor.NAME,
+        new INodeReferenceSectionProcessor());
+    this.sections.put(INodeDirectorySectionProcessor.NAME,
+        new INodeDirectorySectionProcessor());
+    this.sections.put(FilesUnderConstructionSectionProcessor.NAME,
+        new FilesUnderConstructionSectionProcessor());
+    this.sections.put(SnapshotSectionProcessor.NAME,
+        new SnapshotSectionProcessor());
+    this.isoDateFormat = PBImageXmlWriter.createSimpleDateFormat();
+  }
+
+  /**
+   * Read the next tag start or end event.
+   *
+   * @param expected     The name of the next tag we expect.
+   *                     We will validate that the tag has this name,
+   *                     unless this string is enclosed in braces.
+   * @param allowEnd     If true, we will also end events.
+   *                     If false, end events cause an exception.
+   *
+   * @return             The next tag start or end event.
+   */
+  private XMLEvent expectTag(String expected, boolean allowEnd)
+      throws IOException {
+    XMLEvent ev = null;
+    while (true) {
+      try {
+        ev = events.nextEvent();
+      } catch (XMLStreamException e) {
+        throw new IOException("Expecting " + expected +
+            ", but got XMLStreamException", e);
+      }
+      switch (ev.getEventType()) {
+      case XMLEvent.ATTRIBUTE:
+        throw new IOException("Got unexpected attribute: " + ev);
+      case XMLEvent.CHARACTERS:
+        if (!ev.asCharacters().isWhiteSpace()) {
+          throw new IOException("Got unxpected characters while " +
+              "looking for " + expected + ": " +
+              ev.asCharacters().getData());
+        }
+        break;
+      case XMLEvent.END_ELEMENT:
+        if (!allowEnd) {
+          throw new IOException("Got unexpected end event " +
+              "while looking for " + expected);
+        }
+        return ev;
+      case XMLEvent.START_ELEMENT:
+        if (!expected.startsWith("[")) {
+          if (!ev.asStartElement().getName().getLocalPart().
+                equals(expected)) {
+            throw new IOException("Failed to find <" + expected + ">; " +
+                "got " + ev.asStartElement().getName().getLocalPart() +
+                " instead.");
+          }
+        }
+        return ev;
+      default:
+        // Ignore other event types like comment, etc.
+        if (LOG.isTraceEnabled()) {
+          LOG.trace("Skipping XMLEvent of type " +
+              ev.getEventType() + "(" +  ev + ")");
+        }
+        break;
+      }
+    }
+  }
+
+  private void expectTagEnd(String expected) throws IOException {
+    XMLEvent ev = expectTag(expected, true);
+    if (ev.getEventType() != XMLStreamConstants.END_ELEMENT) {
+      throw new IOException("Expected tag end event for " + expected +
+            ", but got: " + ev);
+    }
+    if (!expected.startsWith("[")) {
+      String tag = ev.asEndElement().getName().getLocalPart();
+      if (!tag.equals(expected)) {
+        throw new IOException("Expected tag end event for " + expected +
+        ", but got tag end event for " + tag);
+      }
+    }
+  }
+
+  private static class Node {
+    private static final String EMPTY = "";
+    HashMap<String, LinkedList<Node>> children;
+    String val = EMPTY;
+
+    void addChild(String key, Node node) {
+      if (children == null) {
+        children = new HashMap<>();
+      }
+      LinkedList<Node> cur = children.get(key);
+      if (cur == null) {
+        cur = new LinkedList<>();
+        children.put(key, cur);
+      }
+      cur.add(node);
+    }
+
+    Node removeChild(String key) {
+      if (children == null) {
+        return null;
+      }
+      LinkedList<Node> cur = children.get(key);
+      if (cur == null) {
+        return null;
+      }
+      Node node = cur.remove();
+      if ((node == null) || cur.isEmpty()) {
+        children.remove(key);
+      }
+      return node;
+    }
+
+    String removeChildStr(String key) {
+      Node child = removeChild(key);
+      if (child == null) {
+        return null;
+      }
+      if ((child.children != null) && (!child.children.isEmpty())) {
+        throw new RuntimeException("Node " + key + " contains children " +
+            "of its own.");
+      }
+      return child.getVal();
+    }
+
+    Integer removeChildInt(String key) throws IOException {
+      String str = removeChildStr(key);
+      if (str == null) {
+        return null;
+      }
+      return Integer.valueOf(str);
+    }
+
+    Long removeChildLong(String key) throws IOException {
+      String str = removeChildStr(key);
+      if (str == null) {
+        return null;
+      }
+      return Long.valueOf(str);
+    }
+
+    boolean removeChildBool(String key) throws IOException {
+      String str = removeChildStr(key);
+      if (str == null) {
+        return false;
+      }
+      return true;
+    }
+
+    String getRemainingKeyNames() {
+      if (children == null) {
+        return "";
+      }
+      return StringUtils.join(", ", children.keySet());
+    }
+
+    void verifyNoRemainingKeys(String sectionName) throws IOException {
+      String remainingKeyNames = getRemainingKeyNames();
+      if (!remainingKeyNames.isEmpty()) {
+        throw new IOException("Found unknown XML keys in " +
+            sectionName + ": " + remainingKeyNames);
+      }
+    }
+
+    void setVal(String val) {
+      this.val = val;
+    }
+
+    String getVal() {
+      return val;
+    }
+
+    String dump() {
+      StringBuilder bld = new StringBuilder();
+      if ((children != null) && (!children.isEmpty())) {
+        bld.append("{");
+      }
+      if (val != null) {
+        bld.append("[").append(val).append("]");
+      }
+      if ((children != null) && (!children.isEmpty())) {
+        String prefix = "";
+        for (Map.Entry<String, LinkedList<Node>> entry : children.entrySet()) {
+          for (Node n : entry.getValue()) {
+            bld.append(prefix);
+            bld.append(entry.getKey()).append(": ");
+            bld.append(n.dump());
+            prefix = ", ";
+          }
+        }
+        bld.append("}");
+      }
+      return bld.toString();
+    }
+  }
+
+  private void loadNodeChildrenHelper(Node parent, String expected,
+                                String terminators[]) throws IOException {
+    XMLEvent ev = null;
+    while (true) {
+      try {
+        ev = events.peek();
+        switch (ev.getEventType()) {
+        case XMLEvent.END_ELEMENT:
+          if (terminators.length != 0) {
+            return;
+          }
+          events.nextEvent();
+          return;
+        case XMLEvent.START_ELEMENT:
+          String key = ev.asStartElement().getName().getLocalPart();
+          for (String terminator : terminators) {
+            if (terminator.equals(key)) {
+              return;
+            }
+          }
+          events.nextEvent();
+          Node node = new Node();
+          parent.addChild(key, node);
+          loadNodeChildrenHelper(node, expected, new String[0]);
+          break;
+        case XMLEvent.CHARACTERS:
+          String val = XMLUtils.
+              unmangleXmlString(ev.asCharacters().getData(), true);
+          parent.setVal(val);
+          events.nextEvent();
+          break;
+        case XMLEvent.ATTRIBUTE:
+          throw new IOException("Unexpected XML event " + ev);
+        default:
+          // Ignore other event types like comment, etc.
+          if (LOG.isTraceEnabled()) {
+            LOG.trace("Skipping XMLEvent " + ev);
+          }
+          events.nextEvent();
+          break;
+        }
+      } catch (XMLStreamException e) {
+        throw new IOException("Expecting " + expected +
+            ", but got XMLStreamException", e);
+      }
+    }
+  }
+
+  /**
+   * Load a subtree of the XML into a Node structure.
+   * We will keep consuming XML events until we exit the current subtree.
+   * If there are any terminators specified, we will always leave the
+   * terminating end tag event in the stream.
+   *
+   * @param parent         The node to fill in.
+   * @param expected       A string to display in exceptions.
+   * @param terminators    Entering any one of these XML tags terminates our
+   *                       traversal.
+   * @throws IOException
+   */
+  private void loadNodeChildren(Node parent, String expected,
+                                String... terminators) throws IOException {
+    loadNodeChildrenHelper(parent, expected, terminators);
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("loadNodeChildren(expected=" + expected +
+          ", terminators=[" + StringUtils.join(",", terminators) +
+          "]):" + parent.dump());
+    }
+  }
+
+  /**
+   * A processor for an FSImage XML section.
+   */
+  private interface SectionProcessor {
+    /**
+     * Process this section.
+     */
+    void process() throws IOException;
+  }
+
+  /**
+   * Processes the NameSection containing last allocated block ID, etc.
+   */
+  private class NameSectionProcessor implements SectionProcessor {
+    static final String NAME = "NameSection";
+
+    @Override
+    public void process() throws IOException {
+      Node node = new Node();
+      loadNodeChildren(node, "NameSection fields");
+      NameSystemSection.Builder b =  NameSystemSection.newBuilder();
+      Integer namespaceId = node.removeChildInt("namespaceId");
+      if (namespaceId == null)  {
+        throw new IOException("<NameSection> is missing <namespaceId>");
+      }
+      b.setNamespaceId(namespaceId);
+      Long lval = node.removeChildLong("genstampV1");
+      if (lval != null)  {
+        b.setGenstampV1(lval);
+      }
+      lval = node.removeChildLong("genstampV2");
+      if (lval != null)  {
+        b.setGenstampV2(lval);
+      }
+      lval = node.removeChildLong("genstampV1Limit");
+      if (lval != null)  {
+        b.setGenstampV1Limit(lval);
+      }
+      lval = node.removeChildLong("lastAllocatedBlockId");
+      if (lval != null)  {
+        b.setLastAllocatedBlockId(lval);
+      }
+      lval = node.removeChildLong("txid");
+      if (lval != null)  {
+        b.setTransactionId(lval);
+      }
+      lval = node.removeChildLong("rollingUpgradeStartTime");
+      if (lval != null)  {
+        b.setRollingUpgradeStartTime(lval);
+      }
+      lval = node.removeChildLong("lastAllocatedStripedBlockId");
+      if (lval != null)  {
+        b.setLastAllocatedStripedBlockId(lval);
+      }
+      node.verifyNoRemainingKeys("NameSection");
+      NameSystemSection s = b.build();
+      if (LOG.isDebugEnabled()) {
+        LOG.debug(SectionName.NS_INFO.name() + " writing header: {" +
+            TextFormat.printToString(s) + "}");
+      }
+      s.writeDelimitedTo(out);
+      recordSectionLength(SectionName.NS_INFO.name());
+    }
+  }
+
+  private class INodeSectionProcessor implements SectionProcessor {
+    static final String NAME = "INodeSection";
+
+    @Override
+    public void process() throws IOException {
+      Node headerNode = new Node();
+      loadNodeChildren(headerNode, "INodeSection fields", "inode");
+      INodeSection.Builder b =  INodeSection.newBuilder();
+      Long lval = headerNode.removeChildLong("lastInodeId");
+      if (lval != null)  {
+        b.setLastInodeId(lval);
+      }
+      Integer expectedNumINodes = headerNode.removeChildInt("numInodes");
+      if (expectedNumINodes == null) {
+        throw new IOException("Failed to find <numInodes> in INodeSection.");
+      }
+      b.setNumInodes(expectedNumINodes);
+      INodeSection s = b.build();
+      s.writeDelimitedTo(out);
+      headerNode.verifyNoRemainingKeys("INodeSection");
+      int actualNumINodes = 0;
+      while (actualNumINodes < expectedNumINodes) {
+        try {
+          expectTag("inode", false);
+        } catch (IOException e) {
+          throw new IOException("Only found " + actualNumINodes +
+              " <inode> entries out of " + expectedNumINodes, e);
+        }
+        actualNumINodes++;
+        Node inode = new Node();
+        loadNodeChildren(inode, "INode fields");
+        INodeSection.INode.Builder inodeBld = processINodeXml(inode);
+        inodeBld.build().writeDelimitedTo(out);
+      }
+      expectTagEnd("INodeSection");
+      recordSectionLength(SectionName.INODE.name());
+    }
+  }
+
+  private INodeSection.INode.Builder processINodeXml(Node node)
+      throws IOException {
+    String type = node.removeChildStr("type");
+    if (type == null) {
+      throw new IOException("INode XML found with no <type> tag.");
+    }
+    INodeSection.INode.Builder inodeBld = INodeSection.INode.newBuilder();
+    Long id = node.removeChildLong("id");
+    if (id == null) {
+      throw new IOException("<inode> found without <id>");
+    }
+    inodeBld.setId(id);
+    String name = node.removeChildStr("name");
+    if (name != null) {
+      inodeBld.setName(ByteString.copyFrom(name, "UTF8"));
+    }
+    switch (type) {
+    case "FILE":
+      processFileXml(node, inodeBld );
+      break;
+    case "DIRECTORY":
+      processDirectoryXml(node, inodeBld);
+      break;
+    case "SYMLINK":
+      processSymlinkXml(node, inodeBld);
+      break;
+    default:
+      throw new IOException("INode XML found with unknown <type> " +
+          "tag " + type);
+    }
+    node.verifyNoRemainingKeys("inode");
+    return inodeBld;
+  }
+
+  private void processFileXml(Node node, INodeSection.INode.Builder inodeBld)
+      throws IOException {
+    inodeBld.setType(INodeSection.INode.Type.FILE);
+    INodeSection.INodeFile.Builder bld = INodeSection.INodeFile.newBuilder();
+    Integer ival = node.removeChildInt("replication");
+    if (ival != null) {
+      bld.setReplication(ival);
+    }
+    Long lval = node.removeChildLong("mtime");
+    if (lval != null) {
+      bld.setModificationTime(lval);
+    }
+    lval = node.removeChildLong("atime");
+    if (lval != null) {
+      bld.setAccessTime(lval);
+    }
+    lval = node.removeChildLong("preferredBlockSize");
+    if (lval != null) {
+      bld.setPreferredBlockSize(lval);
+    }
+    String perm = node.removeChildStr("permission");
+    if (perm != null) {
+      bld.setPermission(permissionXmlToU64(perm));
+    }
+    Node blocks = node.removeChild("blocks");
+    if (blocks != null) {
+      while (true) {
+        Node block = blocks.removeChild("block");
+        if (block == null) {
+          break;
+        }
+        HdfsProtos.BlockProto.Builder blockBld =
+            HdfsProtos.BlockProto.newBuilder();
+        Long id = block.removeChildLong("id");
+        if (id == null) {
+          throw new IOException("<block> found without <id>");
+        }
+        blockBld.setBlockId(id);
+        Long genstamp = block.removeChildLong("genstamp");
+        if (genstamp == null) {
+          throw new IOException("<block> found without <genstamp>");
+        }
+        blockBld.setGenStamp(genstamp);
+        Long numBytes = block.removeChildLong("numBytes");
+        if (numBytes == null) {
+          throw new IOException("<block> found without <numBytes>");
+        }
+        blockBld.setNumBytes(numBytes);
+        bld.addBlocks(blockBld);
+      }
+    }
+    Node fileUnderConstruction = node.removeChild("file-under-construction");
+    if (fileUnderConstruction != null) {
+      INodeSection.FileUnderConstructionFeature.Builder fb =
+          INodeSection.FileUnderConstructionFeature.newBuilder();
+      String clientName =
+          fileUnderConstruction.removeChildStr("clientName");
+      if (clientName == null) {
+        throw new IOException("<file-under-construction> found without " +
+            "<clientName>");
+      }
+      fb.setClientName(clientName);
+      String clientMachine =
+          fileUnderConstruction.removeChildStr("clientMachine");
+      if (clientMachine == null) {
+        throw new IOException("<file-under-construction> found without " +
+            "<clientMachine>");
+      }
+      fb.setClientMachine(clientMachine);
+      bld.setFileUC(fb);
+    }
+    Node acls = node.removeChild("acls");
+    if (acls != null) {
+      bld.setAcl(aclXmlToProto(acls));
+    }
+    Node xattrs = node.removeChild("xattrs");
+    if (xattrs != null) {
+      bld.setXAttrs(xattrsXmlToProto(xattrs));
+    }
+    ival = node.removeChildInt("storagePolicyId");
+    if (ival != null) {
+      bld.setStoragePolicyID(ival);
+    }
+    Boolean bval = node.removeChildBool("isStriped");
+    bld.setIsStriped(bval);
+    inodeBld.setFile(bld);
+    // Will check remaining keys and serialize in processINodeXml
+  }
+
+  private void processDirectoryXml(Node node,
+          INodeSection.INode.Builder inodeBld) throws IOException {
+    inodeBld.setType(INodeSection.INode.Type.DIRECTORY);
+    INodeSection.INodeDirectory.Builder bld =
+        INodeSection.INodeDirectory.newBuilder();
+    Long lval = node.removeChildLong("mtime");
+    if (lval != null) {
+      bld.setModificationTime(lval);
+    }
+    lval = node.removeChildLong("nsquota");
+    if (lval != null) {
+      bld.setNsQuota(lval);
+    }
+    lval = node.removeChildLong("dsquota");
+    if (lval != null) {
+      bld.setDsQuota(lval);
+    }
+    String perm = node.removeChildStr("permission");
+    if (perm != null) {
+      bld.setPermission(permissionXmlToU64(perm));
+    }
+    Node acls = node.removeChild("acls");
+    if (acls != null) {
+      bld.setAcl(aclXmlToProto(acls));
+    }
+    Node xattrs = node.removeChild("xattrs");
+    if (xattrs != null) {
+      bld.setXAttrs(xattrsXmlToProto(xattrs));
+    }
+    INodeSection.QuotaByStorageTypeFeatureProto.Builder qf =
+        INodeSection.QuotaByStorageTypeFeatureProto.newBuilder();
+    while (true) {
+      Node typeQuota = node.removeChild("typeQuota");
+      if (typeQuota == null) {
+        break;
+      }
+      INodeSection.QuotaByStorageTypeEntryProto.Builder qbld =
+          INodeSection.QuotaByStorageTypeEntryProto.newBuilder();
+      String type = typeQuota.removeChildStr("type");
+      if (type == null) {
+        throw new IOException("<typeQuota> was missing <type>");
+      }
+      HdfsProtos.StorageTypeProto storageType =
+          HdfsProtos.StorageTypeProto.valueOf(type);
+      if (storageType == null) {
+        throw new IOException("<typeQuota> had unknown <type> " + type);
+      }
+      qbld.setStorageType(storageType);
+      Long quota = typeQuota.removeChildLong("quota");
+      if (quota == null) {
+        throw new IOException("<typeQuota> was missing <quota>");
+      }
+      qbld.setQuota(quota);
+      qf.addQuotas(qbld);
+    }
+    bld.setTypeQuotas(qf);
+    inodeBld.setDirectory(bld);
+    // Will check remaining keys and serialize in processINodeXml
+  }
+
+  private void processSymlinkXml(Node node,
+                                 INodeSection.INode.Builder inodeBld) throws IOException {
+    inodeBld.setType(INodeSection.INode.Type.SYMLINK);
+    INodeSection.INodeSymlink.Builder bld =
+        INodeSection.INodeSymlink.newBuilder();
+    String perm = node.removeChildStr("permission");
+    if (perm != null) {
+      bld.setPermission(permissionXmlToU64(perm));
+    }
+    String target = node.removeChildStr("target");
+    if (target != null) {
+      bld.setTarget(ByteString.copyFrom(target, "UTF8"));
+    }
+    Long lval = node.removeChildLong("mtime");
+    if (lval != null) {
+      bld.setModificationTime(lval);
+    }
+    lval = node.removeChildLong("atime");
+    if (lval != null) {
+      bld.setAccessTime(lval);
+    }
+    inodeBld.setSymlink(bld);
+    // Will check remaining keys and serialize in processINodeXml
+  }
+
+  private INodeSection.AclFeatureProto.Builder aclXmlToProto(Node acl)
+      throws IOException {
+    // TODO: support ACLs
+    throw new IOException("ACLs are not supported yet.");
+  }
+
+  private INodeSection.XAttrFeatureProto.Builder xattrsXmlToProto(Node xattrs)
+      throws IOException {
+    INodeSection.XAttrFeatureProto.Builder bld =
+        INodeSection.XAttrFeatureProto.newBuilder();
+    while (true) {
+      Node xattr = xattrs.removeChild("xattr");
+      if (xattr == null) {
+        break;
+      }
+      INodeSection.XAttrCompactProto.Builder b =
+          INodeSection.XAttrCompactProto.newBuilder();
+      String ns = xattr.removeChildStr("ns");
+      if (ns == null) {
+        throw new IOException("<xattr> had no <ns> entry.");
+      }
+      int nsIdx = XAttrProtos.XAttrProto.
+          XAttrNamespaceProto.valueOf(ns).ordinal();
+      String name = xattr.removeChildStr("name");
+      String valStr = xattr.removeChildStr("val");
+      byte[] val = null;
+      if (valStr == null) {
+        String valHex = xattr.removeChildStr("valHex");
+        if (valHex == null) {
+          throw new IOException("<xattr> had no <val> or <valHex> entry.");
+        }
+        val = new HexBinaryAdapter().unmarshal(valHex);
+      } else {
+        val = valStr.getBytes("UTF8");
+      }
+      b.setValue(ByteString.copyFrom(val));
+
+      // The XAttrCompactProto name field uses a fairly complex format
+      // to encode both the string table ID of the xattr name and the
+      // namespace ID.  See the protobuf file for details.
+      int nameId = registerStringId(name);
+      int encodedName = (nameId << XATTR_NAME_OFFSET) |
+          ((nsIdx & XATTR_NAMESPACE_MASK) << XATTR_NAMESPACE_OFFSET) |
+          (((nsIdx >> 2) & XATTR_NAMESPACE_EXT_MASK)
+              << XATTR_NAMESPACE_EXT_OFFSET);
+      b.setName(encodedName);
+      xattr.verifyNoRemainingKeys("xattr");
+      bld.addXAttrs(b);
+    }
+    xattrs.verifyNoRemainingKeys("xattrs");
+    return bld;
+  }
+
+  private class SecretManagerSectionProcessor implements SectionProcessor {
+    static final String NAME = "SecretManagerSection";
+
+    @Override
+    public void process() throws IOException {
+      Node secretHeader = new Node();
+      loadNodeChildren(secretHeader, "SecretManager fields",
+          "delegationKey", "token");
+      SecretManagerSection.Builder b =  SecretManagerSection.newBuilder();
+      Integer currentId = secretHeader.removeChildInt("currentId");
+      if (currentId == null) {
+        throw new IOException("SecretManager section had no <currentId>");
+      }
+      b.setCurrentId(currentId);
+      Integer tokenSequenceNumber = secretHeader.removeChildInt("tokenSequenceNumber");
+      if (tokenSequenceNumber == null) {
+        throw new IOException("SecretManager section had no " +
+            "<tokenSequenceNumber>");
+      }
+      b.setTokenSequenceNumber(tokenSequenceNumber);
+      Integer expectedNumKeys = secretHeader.removeChildInt("numDelegationKeys");
+      if (expectedNumKeys == null) {
+        throw new IOException("SecretManager section had no " +
+            "<numDelegationKeys>");
+      }
+      b.setNumKeys(expectedNumKeys);
+      Integer expectedNumTokens = secretHeader.removeChildInt("numTokens");
+      if (expectedNumTokens == null) {
+        throw new IOException("SecretManager section had no " +
+            "<numTokens>");
+      }
+      b.setNumTokens(expectedNumTokens);
+      secretHeader.verifyNoRemainingKeys("SecretManager");
+      b.build().writeDelimitedTo(out);
+      for (int actualNumKeys = 0; actualNumKeys < expectedNumKeys;
+           actualNumKeys++) {
+        try {
+          expectTag("delegationKey", false);
+        } catch (IOException e) {
+          throw new IOException("Only read " + actualNumKeys +
+              " delegation keys out of " + expectedNumKeys, e);
+        }
+        SecretManagerSection.DelegationKey.Builder dbld =
+            SecretManagerSection.DelegationKey.newBuilder();
+        Node dkey = new Node();
+        loadNodeChildren(dkey, "Delegation key fields");
+        Integer id = dkey.removeChildInt("id");
+        if (id == null) {
+          throw new IOException("Delegation key stanza <delegationKey> " +
+              "lacked an <id> field.");
+        }
+        dbld.setId(id);
+        String expiry = dkey.removeChildStr("expiry");
+        if (expiry == null) {
+          throw new IOException("Delegation key stanza <delegationKey> " +
+              "lacked an <expiry> field.");
+        }
+        dbld.setExpiryDate(dateStrToLong(expiry));
+        String keyHex = dkey.removeChildStr("key");
+        if (keyHex == null) {
+          throw new IOException("Delegation key stanza <delegationKey> " +
+              "lacked a <key> field.");
+        }
+        byte[] key = new HexBinaryAdapter().unmarshal(keyHex);
+        dkey.verifyNoRemainingKeys("delegationKey");
+        dbld.setKey(ByteString.copyFrom(key));
+        dbld.build().writeDelimitedTo(out);
+      }
+      for (int actualNumTokens = 0; actualNumTokens < expectedNumTokens;
+           actualNumTokens++) {
+        try {
+          expectTag("token", false);
+        } catch (IOException e) {
+          throw new IOException("Only read " + actualNumTokens +
+              " tokens out of " + expectedNumTokens, e);
+        }
+        SecretManagerSection.PersistToken.Builder tbld =
+            SecretManagerSection.PersistToken.newBuilder();
+        Node token = new Node();
+        loadNodeChildren(token, "PersistToken key fields");
+        Integer version = token.removeChildInt("version");
+        if (version != null) {
+          tbld.setVersion(version);
+        }
+        String owner = token.removeChildStr("owner");
+        if (owner != null) {
+          tbld.setOwner(owner);
+        }
+        String renewer = token.removeChildStr("renewer");
+        if (renewer != null) {
+          tbld.setRenewer(renewer);
+        }
+        String realUser = token.removeChildStr("realUser");
+        if (realUser != null) {
+          tbld.setRealUser(realUser);
+        }
+        String issueDateStr = token.removeChildStr("issueDate");
+        if (issueDateStr != null) {
+          tbld.setIssueDate(dateStrToLong(issueDateStr));
+        }
+        String maxDateStr = token.removeChildStr("maxDate");
+        if (maxDateStr != null) {
+          tbld.setMaxDate(dateStrToLong(maxDateStr));
+        }
+        Integer seqNo = token.removeChildInt("sequenceNumber");
+        if (seqNo != null) {
+          tbld.setSequenceNumber(seqNo);
+        }
+        Integer masterKeyId = token.removeChildInt("masterKeyId");
+        if (masterKeyId != null) {
+          tbld.setMasterKeyId(masterKeyId);
+        }
+        String expiryDateStr = token.removeChildStr("expiryDate");
+        if (expiryDateStr != null) {
+          tbld.setExpiryDate(dateStrToLong(expiryDateStr));
+        }
+        token.verifyNoRemainingKeys("token");
+        tbld.build().writeDelimitedTo(out);
+      }
+      expectTagEnd("SecretManagerSection");
+      recordSectionLength(SectionName.SECRET_MANAGER.name());
+    }
+
+    private long dateStrToLong(String dateStr) throws IOException {
+      try {
+        Date date = isoDateFormat.parse(dateStr);
+        return date.getTime();
+      } catch (ParseException e) {
+        throw new IOException("Failed to parse ISO date string " + dateStr, e);
+      }
+    }
+  }
+
+  private class CacheManagerSectionProcessor implements SectionProcessor {
+    static final String NAME = "CacheManagerSection";
+
+    @Override
+    public void process() throws IOException {
+      Node node = new Node();
+      loadNodeChildren(node, "CacheManager fields", "pool", "directive");
+      CacheManagerSection.Builder b =  CacheManagerSection.newBuilder();
+      Long nextDirectiveId = node.removeChildLong("nextDirectiveId");
+      if (nextDirectiveId == null) {
+        throw new IOException("CacheManager section had no <nextDirectiveId>");
+      }
+      b.setNextDirectiveId(nextDirectiveId);
+      Integer expectedNumPools = node.removeChildInt("numPools");
+      if (expectedNumPools == null) {
+        throw new IOException("CacheManager section had no <numPools>");
+      }
+      b.setNumPools(expectedNumPools);
+      Integer expectedNumDirectives = node.removeChildInt("numDirectives");
+      if (expectedNumDirectives == null) {
+        throw new IOException("CacheManager section had no <numDirectives>");
+      }
+      b.setNumDirectives(expectedNumDirectives);
+      b.build().writeDelimitedTo(out);
+      long actualNumPools = 0;
+      while (actualNumPools < expectedNumPools) {
+        try {
+          expectTag("pool", false);
+        } catch (IOException e) {
+          throw new IOException("Only read " + actualNumPools +
+              " cache pools out of " + expectedNumPools, e);
+        }
+        actualNumPools++;
+        Node pool = new Node();
+        loadNodeChildren(pool, "pool fields", "");
+        processPoolXml(node);
+      }
+      long actualNumDirectives = 0;
+      while (actualNumDirectives < expectedNumDirectives) {
+        try {
+          expectTag("directive", false);
+        } catch (IOException e) {
+          throw new IOException("Only read " + actualNumDirectives +
+              " cache pools out of " + expectedNumDirectives, e);
+        }
+        actualNumDirectives++;
+        Node pool = new Node();
+        loadNodeChildren(pool, "directive fields", "");
+        processDirectiveXml(node);
+      }
+      expectTagEnd("CacheManagerSection");
+      recordSectionLength(SectionName.CACHE_MANAGER.name());
+    }
+
+    private void processPoolXml(Node pool) throws IOException {
+      CachePoolInfoProto.Builder bld = CachePoolInfoProto.newBuilder();
+      String poolName = pool.removeChildStr("poolName");
+      if (poolName == null) {
+        throw new IOException("<pool> found without <poolName>");
+      }
+      bld.setPoolName(poolName);
+      String ownerName = pool.removeChildStr("ownerName");
+      if (ownerName == null) {
+        throw new IOException("<pool> found without <ownerName>");
+      }
+      bld.setOwnerName(ownerName);
+      String groupName = pool.removeChildStr("groupName");
+      if (groupName == null) {
+        throw new IOException("<pool> found without <groupName>");
+      }
+      bld.setGroupName(groupName);
+      Integer mode = pool.removeChildInt("mode");
+      if (mode == null) {
+        throw new IOException("<pool> found without <mode>");
+      }
+      bld.setMode(mode);
+      Long limit = pool.removeChildLong("limit");
+      if (limit == null) {
+        throw new IOException("<pool> found without <limit>");
+      }
+      bld.setLimit(limit);
+      Long maxRelativeExpiry = pool.removeChildLong("maxRelativeExpiry");
+      if (maxRelativeExpiry == null) {
+        throw new IOException("<pool> found without <maxRelativeExpiry>");
+      }
+      bld.setMaxRelativeExpiry(maxRelativeExpiry);
+      pool.verifyNoRemainingKeys("pool");
+      bld.build().writeDelimitedTo(out);
+    }
+
+    private void processDirectiveXml(Node directive) throws IOException {
+      CacheDirectiveInfoProto.Builder bld =
+          CacheDirectiveInfoProto.newBuilder();
+      Long id = directive.removeChildLong("id");
+      if (id == null) {
+        throw new IOException("<directive> found without <id>");
+      }
+      bld.setId(id);
+      String path = directive.removeChildStr("path");
+      if (path == null) {
+        throw new IOException("<directive> found without <path>");
+      }
+      bld.setPath(path);
+      Integer replication = directive.removeChildInt("replication");
+      if (replication == null) {
+        throw new IOException("<directive> found without <replication>");
+      }
+      bld.setReplication(replication);
+      String pool = directive.removeChildStr("pool");
+      if (path == null) {
+        throw new IOException("<directive> found without <pool>");
+      }
+      bld.setPool(pool);
+      Node expiration = directive.removeChild("expiration");
+      if (expiration != null) {
+        CacheDirectiveInfoExpirationProto.Builder ebld =
+            CacheDirectiveInfoExpirationProto.newBuilder();
+        Long millis = expiration.removeChildLong("millis");
+        if (millis == null) {
+          throw new IOException("cache directive <expiration> found " +
+              "without <millis>");
+        }
+        ebld.setMillis(millis);
+        if (expiration.removeChildBool("relative")) {
+          ebld.setIsRelative(true);
+        } else {
+          ebld.setIsRelative(false);
+        }
+        bld.setExpiration(ebld);
+      }
+      directive.verifyNoRemainingKeys("directive");
+      bld.build().writeDelimitedTo(out);
+    }
+  }
+
+  private class INodeReferenceSectionProcessor implements SectionProcessor {
+    static final String NAME = "INodeReferenceSection";
+
+    @Override
+    public void process() throws IOException {
+      // There is no header for this section.
+      // We process the repeated <ref> elements.
+      while (true) {
+        XMLEvent ev = expectTag("ref", true);
+        if (ev.isEndElement()) {
+          break;
+        }
+        Node inodeRef = new Node();
+        FsImageProto.INodeReferenceSection.INodeReference.Builder bld =
+            FsImageProto.INodeReferenceSection.INodeReference.newBuilder();
+        loadNodeChildren(inodeRef, "INodeReference");
+        Long referredId = inodeRef.removeChildLong("referredId");
+        if (referredId != null) {
+          bld.setReferredId(referredId);
+        }
+        String name = inodeRef.removeChildStr("name");
+        if (name != null) {
+          bld.setName(ByteString.copyFrom(name, "UTF8"));
+        }
+        Integer dstSnapshotId = inodeRef.removeChildInt("dstSnapshotId");
+        if (dstSnapshotId != null) {
+          bld.setDstSnapshotId(dstSnapshotId);
+        }
+        Integer lastSnapshotId = inodeRef.removeChildInt("lastSnapshotId");
+        if (lastSnapshotId != null) {
+          bld.setLastSnapshotId(lastSnapshotId);
+        }
+        inodeRef.verifyNoRemainingKeys("ref");
+        bld.build().writeDelimitedTo(out);
+      }
+      recordSectionLength(SectionName.INODE_REFERENCE.name());
+    }
+  }
+
+  private class INodeDirectorySectionProcessor implements SectionProcessor {
+    static final String NAME = "INodeDirectorySection";
+
+    @Override
+    public void process() throws IOException {
+      // No header for this section
+      // Process the repeated <directory> elements.
+      while (true) {
+        XMLEvent ev = expectTag("directory", true);
+        if (ev.isEndElement()) {
+          break;
+        }
+        Node directory = new Node();
+        FsImageProto.INodeDirectorySection.DirEntry.Builder bld =
+            FsImageProto.INodeDirectorySection.DirEntry.newBuilder();
+        loadNodeChildren(directory, "directory");
+        Long parent = directory.removeChildLong("parent");
+        if (parent != null) {
+          bld.setParent(parent);
+        }
+        while (true) {
+          Node child = directory.removeChild("child");
+          if (child == null) {
+            break;
+          }
+          bld.addChildren(Long.parseLong(child.getVal()));
+        }
+        while (true) {
+          Node refChild = directory.removeChild("refChild");
+          if (refChild == null) {
+            break;
+          }
+          bld.addRefChildren(Integer.parseInt(refChild.getVal()));
+        }
+        directory.verifyNoRemainingKeys("directory");
+        bld.build().writeDelimitedTo(out);
+      }
+      recordSectionLength(SectionName.INODE_DIR.name());
+    }
+  }
+
+  private class FilesUnderConstructionSectionProcessor
+      implements SectionProcessor {
+    static final String NAME = "FileUnderConstructionSection";
+
+    @Override
+    public void process() throws IOException {
+      // No header for this section type.
+      // Process the repeated files under construction elements.
+      while (true) {
+        XMLEvent ev = expectTag("inode", true);
+        if (ev.isEndElement()) {
+          break;
+        }
+        Node fileUnderConstruction = new Node();
+        loadNodeChildren(fileUnderConstruction, "file under construction");
+        FileUnderConstructionEntry.Builder bld =
+            FileUnderConstructionEntry.newBuilder();
+        Long id = fileUnderConstruction.removeChildLong("id");
+        if (id != null) {
+          bld.setInodeId(id);
+        }
+        String fullpath = fileUnderConstruction.removeChildStr("path");
+        if (fullpath != null) {
+          bld.setFullPath(fullpath);
+        }
+        fileUnderConstruction.verifyNoRemainingKeys("inode");
+        bld.build().writeDelimitedTo(out);
+      }
+      recordSectionLength(SectionName.FILES_UNDERCONSTRUCTION.name());
+    }
+  }
+
+  private class SnapshotSectionProcessor implements SectionProcessor {
+    static final String NAME = "SnapshotSection";
+
+    @Override
+    public void process() throws IOException {
+      FsImageProto.SnapshotSection.Builder bld =
+          FsImageProto.SnapshotSection.newBuilder();
+      Node header = new Node();
+      loadNodeChildren(header, "SnapshotSection fields", "snapshot");
+      Integer snapshotCounter = header.removeChildInt("snapshotCounter");
+      if (snapshotCounter == null) {
+        throw new IOException("No <snapshotCounter> entry found in " +
+            "SnapshotSection header");
+      }
+      bld.setSnapshotCounter(snapshotCounter);
+      Integer expectedNumSnapshots = header.removeChildInt("numSnapshots");
+      if (expectedNumSnapshots == null) {
+        throw new IOException("No <numSnapshots> entry found in " +
+            "SnapshotSection header");
+      }
+      bld.setNumSnapshots(expectedNumSnapshots);
+      while (true) {
+        Node sd = header.removeChild("snapshottableDir");
+        if (sd == null) {
+          break;
+        }
+        Long dir = sd.removeChildLong("dir");
+        sd.verifyNoRemainingKeys("<dir>");
+        bld.addSnapshottableDir(dir);
+      }
+      header.verifyNoRemainingKeys("SnapshotSection");
+      bld.build().writeDelimitedTo(out);
+      int actualNumSnapshots = 0;
+      while (actualNumSnapshots < expectedNumSnapshots) {
+        try {
+          expectTag("snapshot", false);
+        } catch (IOException e) {
+          throw new IOException("Only read " + actualNumSnapshots +
+              " <snapshot> entries out of " + expectedNumSnapshots, e);
+        }
+        actualNumSnapshots++;
+        Node snapshot = new Node();
+        loadNodeChildren(snapshot, "snapshot fields");
+        FsImageProto.SnapshotSection.Snapshot.Builder s =
+            FsImageProto.SnapshotSection.Snapshot.newBuilder();
+        Integer snapshotId = snapshot.removeChildInt("id");
+        if (snapshotId == null) {
+          throw new IOException("<snapshot> section was missing <id>");
+        }
+        s.setSnapshotId(snapshotId);
+        Node snapshotRoot = snapshot.removeChild("root");
+        INodeSection.INode.Builder inodeBld = processINodeXml(snapshotRoot);
+        s.setRoot(inodeBld);
+        s.build().writeDelimitedTo(out);
+      }
+      expectTagEnd("SnapshotSection");
+      recordSectionLength(SectionName.SNAPSHOT.name());
+    }
+  }
+
+  private class SnapshotDiffSectionProcessor implements SectionProcessor {
+    static final String NAME = "SnapshotDiffSection";
+
+    @Override
+    public void process() throws IOException {
+      // No header for this section type.
+      LOG.debug("Processing SnapshotDiffSection");
+      while (true) {
+        XMLEvent ev = expectTag("[diff start tag]", true);
+        if (ev.isEndElement()) {
+          String name = ev.asEndElement().getName().getLocalPart();
+          if (name.equals("SnapshotDiffSection")) {
+            break;
+          }
+          throw new IOException("Got unexpected end tag for " + name);
+        }
+        String tagName = ev.asStartElement().getName().getLocalPart();
+        if (tagName.equals("dirDiffEntry")) {
+          processDirDiffEntry();
+        } else if (tagName.equals("fileDiffEntry")) {
+          processFileDiffEntry();
+        } else {
+          throw new IOException("SnapshotDiffSection contained unexpected " +
+              "tag " + tagName);
+        }
+      }
+      recordSectionLength(SectionName.SNAPSHOT_DIFF.name());
+    }
+
+    private void processDirDiffEntry() throws IOException {
+      LOG.debug("Processing dirDiffEntry");
+      DiffEntry.Builder headerBld = DiffEntry.newBuilder();
+      headerBld.setType(DiffEntry.Type.DIRECTORYDIFF);
+      Node dirDiffHeader = new Node();
+      loadNodeChildren(dirDiffHeader, "dirDiffEntry fields", "dirDiff");
+      Long inodeId = dirDiffHeader.removeChildLong("inodeId");
+      if (inodeId == null) {
+        throw new IOException("<dirDiffEntry> contained no <inodeId> entry.");
+      }
+      headerBld.setInodeId(inodeId);
+      Integer expectedDiffs = dirDiffHeader.removeChildInt("count");
+      if (expectedDiffs == null) {
+        throw new IOException("<dirDiffEntry> contained no <count> entry.");
+      }
+      headerBld.setNumOfDiff(expectedDiffs);
+      dirDiffHeader.verifyNoRemainingKeys("dirDiffEntry");
+      headerBld.build().writeDelimitedTo(out);
+      for (int actualDiffs = 0; actualDiffs < expectedDiffs; actualDiffs++) {
+        try {
+          expectTag("dirDiff", false);
+        } catch (IOException e) {
+          throw new IOException("Only read " + (actualDiffs + 1) +
+              " diffs out of " + expectedDiffs, e);
+        }
+        Node dirDiff = new Node();
+        loadNodeChildren(dirDiff, "dirDiff fields");
+        FsImageProto.SnapshotDiffSection.DirectoryDiff.Builder bld =
+            FsImageProto.SnapshotDiffSection.DirectoryDiff.newBuilder();
+        Integer snapshotId = dirDiff.removeChildInt("snapshotId");
+        if (snapshotId != null) {
+          bld.setSnapshotId(snapshotId);
+        }
+        Integer childrenSize = dirDiff.removeChildInt("childrenSize");
+        if (childrenSize == null) {
+          throw new IOException("Expected to find <childrenSize> in " +
+              "<dirDiff> section.");
+        }
+        bld.setIsSnapshotRoot(dirDiff.removeChildBool("isSnapshotRoot"));
+        bld.setChildrenSize(childrenSize);
+        String name = dirDiff.removeChildStr("name");
+        if (name != null) {
+          bld.setName(ByteString.copyFrom(name, "UTF8"));
+        }
+        // TODO: add missing snapshotCopy field to XML
+        Integer expectedCreatedListSize =
+            dirDiff.removeChildInt("createdListSize");
+        if (expectedCreatedListSize == null) {
+          throw new IOException("Expected to find <createdListSize> in " +
+              "<dirDiff> section.");
+        }
+        bld.setCreatedListSize(expectedCreatedListSize);
+        while (true) {
+          Node deleted = dirDiff.removeChild("deletedInode");
+          if (deleted == null){
+            break;
+          }
+          bld.addDeletedINode(Long.parseLong(deleted.getVal()));
+        }
+        while (true) {
+          Node deleted = dirDiff.removeChild("deletedInoderef");
+          if (deleted == null){
+            break;
+          }
+          bld.addDeletedINodeRef(Integer.parseInt(deleted.getVal()));
+        }
+        bld.build().writeDelimitedTo(out);
+        // After the DirectoryDiff header comes a list of CreatedListEntry PBs.
+        int actualCreatedListSize = 0;
+        while (true) {
+          Node created = dirDiff.removeChild("created");
+          if (created == null){
+            break;
+          }
+          String cleName = created.removeChildStr("name");
+          if (cleName == null) {
+            throw new IOException("Expected <created> entry to have " +
+                "a <name> field");
+          }
+          created.verifyNoRemainingKeys("created");
+          FsImageProto.SnapshotDiffSection.CreatedListEntry.newBuilder().
+              setName(ByteString.copyFrom(cleName, "UTF8")).
+              build().writeDelimitedTo(out);
+          actualCreatedListSize++;
+        }
+        if (actualCreatedListSize != expectedCreatedListSize) {
+          throw new IOException("<createdListSize> was " +
+              expectedCreatedListSize +", but there were " +
+              actualCreatedListSize + " <created> entries.");
+        }
+        dirDiff.verifyNoRemainingKeys("dirDiff");
+      }
+      expectTagEnd("dirDiffEntry");
+    }
+
+    private void processFileDiffEntry() throws IOException {
+      LOG.debug("Processing fileDiffEntry");
+      DiffEntry.Builder headerBld = DiffEntry.newBuilder();
+      headerBld.setType(DiffEntry.Type.FILEDIFF);
+      Node fileDiffHeader = new Node();
+      loadNodeChildren(fileDiffHeader, "fileDiffEntry fields", "fileDiff");
+      Long inodeId = fileDiffHeader.removeChildLong("inodeid");
+      if (inodeId == null) {
+        throw new IOException("<fileDiffEntry> contained no <inodeid> entry.");
+      }
+      headerBld.setInodeId(inodeId);
+      Integer expectedDiffs = fileDiffHeader.removeChildInt("count");
+      if (expectedDiffs == null) {
+        throw new IOException("<fileDiffEntry> contained no <count> entry.");
+      }
+      headerBld.setNumOfDiff(expectedDiffs);
+      fileDiffHeader.verifyNoRemainingKeys("fileDiffEntry");
+      headerBld.build().writeDelimitedTo(out);
+      for (int actualDiffs = 0; actualDiffs < expectedDiffs; actualDiffs++) {
+        try {
+          expectTag("fileDiff", false);
+        } catch (IOException e) {
+          throw new IOException("Only read " + (actualDiffs + 1) +
+              " diffs out of " + expectedDiffs, e);
+        }
+        Node fileDiff = new Node();
+        loadNodeChildren(fileDiff, "fileDiff fields");
+        FsImageProto.SnapshotDiffSection.FileDiff.Builder bld =
+            FsImageProto.SnapshotDiffSection.FileDiff.newBuilder();
+        Integer snapshotId = fileDiff.removeChildInt("snapshotId");
+        if (snapshotId != null) {
+          bld.setSnapshotId(snapshotId);
+        }
+        Long size = fileDiff.removeChildLong("size");
+        if (size != null) {
+          bld.setFileSize(size);
+        }
+        String name = fileDiff.removeChildStr("name");
+        if (name != null) {
+          bld.setName(ByteString.copyFrom(name, "UTF8"));
+        }
+        // TODO: missing snapshotCopy
+        // TODO: missing blocks
+        fileDiff.verifyNoRemainingKeys("fileDiff");
+        bld.build().writeDelimitedTo(out);
+      }
+      expectTagEnd("fileDiffEntry");
+    }
+  }
+
+  /**
+   * Permission is serialized as a 64-bit long. [0:24):[25:48):[48:64)
+   * (in Big Endian).  The first and the second parts are the string ids
+   * of the user and group name, and the last 16 bits are the permission bits.
+   *
+   * @param perm           The permission string from the XML.
+   * @return               The 64-bit value to use in the fsimage for permission.
+   * @throws IOException   If we run out of string IDs in the string table.
+   */
+  private long permissionXmlToU64(String perm) throws IOException {
+    String components[] = perm.split(":");
+    if (components.length != 3) {
+      throw new IOException("Unable to parse permission string " + perm +
+          ": expected 3 components, but only had " + components.length);
+    }
+    String userName = components[0];
+    String groupName = components[1];
+    String modeString = components[2];
+    long userNameId = registerStringId(userName);
+    long groupNameId = registerStringId(groupName);
+    long mode = new FsPermission(modeString).toShort();
+    return (userNameId << 40) | (groupNameId << 16) | mode;
+  }
+
+  /**
+   * The FSImage contains a string table which maps strings to IDs.
+   * This is a simple form of compression which takes advantage of the fact
+   * that the same strings tend to occur over and over again.
+   * This function will return an ID which we can use to represent the given
+   * string.  If the string already exists in the string table, we will use
+   * that ID; otherwise, we will allocate a new one.
+   *
+   * @param str           The string.
+   * @return              The ID in the string table.
+   * @throws IOException  If we run out of bits in the string table.  We only
+   *                      have 25 bits.
+   */
+  int registerStringId(String str) throws IOException {
+    Integer id = stringTable.get(str);
+    if (id != null) {
+      return id;
+    }
+    int latestId = latestStringId;
+    if (latestId >= 0x1ffffff) {
+      throw new IOException("Cannot have more than 2**25 " +
+          "strings in the fsimage, because of the limitation on " +
+          "the size of string table IDs.");
+    }
+    stringTable.put(str, latestId);
+    latestStringId++;
+    return latestId;
+  }
+
+  /**
+   * Record the length of a section of the FSImage in our FileSummary object.
+   * The FileSummary appears at the end of the FSImage and acts as a table of
+   * contents for the file.
+   *
+   * @param sectionNamePb  The name of the section as it should appear in
+   *                       the fsimage.  (This is different than the XML
+   *                       name.)
+   * @throws IOException
+   */
+  void recordSectionLength(String sectionNamePb) throws IOException {
+    long curSectionStartOffset = sectionStartOffset;
+    long curPos = out.getCount();
+    //if (sectionNamePb.equals(SectionName.STRING_TABLE.name())) {
+      fileSummaryBld.addSections(FileSummary.Section.newBuilder().
+          setName(sectionNamePb).
+          setLength(curPos - curSectionStartOffset).
+          setOffset(curSectionStartOffset));
+    //}
+    sectionStartOffset = curPos;
+  }
+
+  /**
+   * Read the version tag which starts the XML file.
+   */
+  private void readVersion() throws IOException {
+    try {
+      expectTag("version", false);
+    } catch (IOException e) {
+      // Handle the case where <version> does not exist.
+      // Note: fsimage XML files which are missing <version> are also missing
+      // many other fields that ovi needs to accurately reconstruct the
+      // fsimage.
+      throw new IOException("No <version> section found at the top of " +
+          "the fsimage XML.  This XML file is too old to be processed " +
+          "by ovi.", e);
+    }
+    Node version = new Node();
+    loadNodeChildren(version, "version fields");
+    Integer onDiskVersion = version.removeChildInt("onDiskVersion");
+    if (onDiskVersion == null) {
+      throw new IOException("The <version> section doesn't contain " +
+          "the onDiskVersion.");
+    }
+    Integer layoutVersion = version.removeChildInt("layoutVersion");
+    if (layoutVersion == null) {
+      throw new IOException("The <version> section doesn't contain " +
+          "the layoutVersion.");
+    }
+    fileSummaryBld.setOndiskVersion(onDiskVersion);
+    fileSummaryBld.setLayoutVersion(layoutVersion);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Loaded <version> with onDiskVersion=" + onDiskVersion +
+          ", layoutVersion=" + layoutVersion + ".");
+    }
+  }
+
+  /**
+   * Write the string table to the fsimage.
+   * @throws IOException
+   */
+  private void writeStringTableSection() throws IOException {
+    FsImageProto.StringTableSection sectionHeader =
+        FsImageProto.StringTableSection.newBuilder().
+        setNumEntry(stringTable.size()).build();
+    if (LOG.isDebugEnabled()) {
+      LOG.debug(SectionName.STRING_TABLE.name() + " writing header: {" +
+            TextFormat.printToString(sectionHeader) + "}");
+    }
+    sectionHeader.writeDelimitedTo(out);
+
+    // The entries don't have to be in any particular order, so iterating
+    // over the hash table is fine.
+    for (Map.Entry<String, Integer> entry : stringTable.entrySet()) {
+      FsImageProto.StringTableSection.Entry stEntry =
+          FsImageProto.StringTableSection.Entry.newBuilder().
+          setStr(entry.getKey()).
+          setId(entry.getValue()).
+          build();
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("Writing string table entry: {" +
+            TextFormat.printToString(stEntry) + "}");
+      }
+      stEntry.writeDelimitedTo(out);
+    }
+    recordSectionLength(SectionName.STRING_TABLE.name());
+  }
+
+  /**
+   * Processes the XML file back into an fsimage.
+   */
+  private void processXml() throws Exception {
+    LOG.debug("Loading <fsimage>.");
+    expectTag("fsimage", false);
+    // Read the <version> tag.
+    readVersion();
+    // Write the HDFSIMG1 magic number which begins the fsimage file.
+    out.write(FSImageUtil.MAGIC_HEADER);
+    // Write a series of fsimage sections.
+    sectionStartOffset = FSImageUtil.MAGIC_HEADER.length;
+    final HashSet<String> unprocessedSections =
+        new HashSet<>(sections.keySet());
+    while (!unprocessedSections.isEmpty()) {
+      XMLEvent ev = expectTag("[section header]", true);
+      if (ev.getEventType() == XMLStreamConstants.END_ELEMENT) {
+        if (ev.asEndElement().getName().getLocalPart().equals("fsimage")) {
+          throw new IOException("FSImage XML ended prematurely, without " +
+              "including section(s) " + StringUtils.join(", ",
+              unprocessedSections));
+        }
+        throw new IOException("Got unexpected tag end event for " +
+            ev.asEndElement().getName().getLocalPart() + " while looking " +
+            "for section header tag.");
+      } else if (ev.getEventType() != XMLStreamConstants.START_ELEMENT) {
+        throw new IOException("Expected section header START_ELEMENT; " +
+            "got event of type " + ev.getEventType());
+      }
+      String sectionName = ev.asStartElement().getName().getLocalPart();
+      if (!unprocessedSections.contains(sectionName)) {
+        throw new IOException("Unknown or duplicate section found for " +
+            sectionName);
+      }
+      SectionProcessor sectionProcessor = sections.get(sectionName);
+      if (sectionProcessor == null) {
+        throw new IOException("Unknown FSImage section " + sectionName +
+            ".  Valid section names are [" +
+            StringUtils.join(", ", sections.keySet()) + "]");
+      }
+      unprocessedSections.remove(sectionName);
+      sectionProcessor.process();
+    }
+
+    // Write the StringTable section to disk.
+    // This has to be done after the other sections, since some of them
+    // add entries to the string table.
+    writeStringTableSection();
+
+    // Write the FileSummary section to disk.
+    // This section is always last.
+    long prevOffset = out.getCount();
+    FileSummary fileSummary = fileSummaryBld.build();
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Writing FileSummary: {" +
+          TextFormat.printToString(fileSummary) + "}");
+    }
+    // Even though the last 4 bytes of the file gives the FileSummary length,
+    // we still write a varint first that also contains the length.
+    fileSummary.writeDelimitedTo(out);
+
+    // Write the length of the FileSummary section as a fixed-size big
+    // endian 4-byte quantity.
+    int summaryLen = Ints.checkedCast(out.getCount() - prevOffset);
+    byte[] summaryLenBytes = new byte[4];
+    ByteBuffer.wrap(summaryLenBytes).asIntBuffer().put(summaryLen);
+    out.write(summaryLenBytes);
+  }
+
+  /**
+   * Run the OfflineImageReconstructor.
+   *
+   * @param inputPath         The input path to use.
+   * @param outputPath        The output path to use.
+   *
+   * @throws Exception        On error.
+   */
+  public static void run(String inputPath, String outputPath)
+      throws Exception {
+    MessageDigest digester = MD5Hash.getDigester();
+    FileOutputStream fout = null;
+    File foutHash = new File(outputPath + ".md5");
+    Files.deleteIfExists(foutHash.toPath()); // delete any .md5 file that exists
+    CountingOutputStream out = null;
+    FileInputStream fis = null;
+    InputStreamReader reader = null;
+    try {
+      Files.deleteIfExists(Paths.get(outputPath));
+      fout = new FileOutputStream(outputPath);
+      fis = new FileInputStream(inputPath);
+      reader = new InputStreamReader(fis, Charset.forName("UTF-8"));
+      out = new CountingOutputStream(
+          new DigestOutputStream(
+              new BufferedOutputStream(fout), digester));
+      OfflineImageReconstructor oir =
+          new OfflineImageReconstructor(out, reader);
+      oir.processXml();
+    } finally {
+      IOUtils.cleanup(LOG, reader, fis, out, fout);
+    }
+    // Write the md5 file
+    MD5FileUtils.saveMD5File(new File(outputPath),
+        new MD5Hash(digester.digest()));
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java

@@ -60,6 +60,8 @@ public class OfflineImageViewerPB {
       + "  * XML: This processor creates an XML document with all elements of\n"
       + "    the fsimage enumerated, suitable for further analysis by XML\n"
       + "    tools.\n"
+      + "  * reverseXML: This processor takes an XML file and creates a\n"
+      + "    binary fsimage containing the same elements.\n"
       + "  * FileDistribution: This processor analyzes the file size\n"
       + "    distribution in the image.\n"
       + "    -maxSize specifies the range [0, maxSize] of file sizes to be\n"
@@ -73,15 +75,18 @@ public class OfflineImageViewerPB {
       + "    changed via the -delimiter argument.\n"
       + "\n"
       + "Required command line arguments:\n"
-      + "-i,--inputFile <arg>   FSImage file to process.\n"
+      + "-i,--inputFile <arg>   FSImage or XML file to process.\n"
       + "\n"
       + "Optional command line arguments:\n"
       + "-o,--outputFile <arg>  Name of output file. If the specified\n"
       + "                       file exists, it will be overwritten.\n"
       + "                       (output to stdout by default)\n"
+      + "                       If the input file was an XML file, we\n"
+      + "                       will also create an <outputFile>.md5 file.\n"
       + "-p,--processor <arg>   Select which type of processor to apply\n"
-      + "                       against image file. (XML|FileDistribution|Web|Delimited)\n"
-      + "                       (Web by default)\n"
+      + "                       against image file. (XML|FileDistribution|\n"
+      + "                       ReverseXML|Web|Delimited)\n"
+      + "                       The default is Web.\n"
       + "-delimiter <arg>       Delimiting string to use with Delimited processor.  \n"
       + "-t,--temp <arg>        Use temporary dir to cache intermediate result to generate\n"
       + "                       Delimited outputs. If not set, Delimited processor constructs\n"
@@ -177,6 +182,16 @@ public class OfflineImageViewerPB {
           new PBImageXmlWriter(conf, out).visit(
               new RandomAccessFile(inputFile, "r"));
           break;
+        case "ReverseXML":
+          try {
+            OfflineImageReconstructor.run(inputFile, outputFile);
+          } catch (Exception e) {
+            System.err.println("OfflineImageReconstructor failed: " +
+                e.getMessage());
+            e.printStackTrace(System.err);
+            System.exit(1);
+          }
+          break;
         case "Web":
           String addr = cmd.getOptionValue("addr", "localhost:5978");
           try (WebImageViewer viewer = new WebImageViewer(
@@ -200,6 +215,7 @@ public class OfflineImageViewerPB {
       System.err.println("Input file ended unexpectedly. Exiting");
     } catch (IOException e) {
       System.err.println("Encountered exception.  Exiting: " + e.getMessage());
+      e.printStackTrace(System.err);
     }
     return -1;
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java

@@ -23,17 +23,24 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintStream;
 import java.io.RandomAccessFile;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.Date;
+import java.util.TimeZone;
 
+import com.google.protobuf.ByteString;
+import org.apache.commons.codec.binary.Hex;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoExpirationProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos;
 import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode;
 import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName;
 import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
@@ -55,6 +62,14 @@ import org.apache.hadoop.hdfs.util.XMLUtils;
 import org.apache.hadoop.util.LimitInputStream;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
+import org.apache.hadoop.util.VersionInfo;
+
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_MASK;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_OFFSET;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_EXT_MASK;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_EXT_OFFSET;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAME_OFFSET;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAME_MASK;
 
 /**
  * PBImageXmlWriter walks over an fsimage structure and writes out
@@ -64,11 +79,20 @@ import com.google.common.collect.Lists;
 public final class PBImageXmlWriter {
   private final Configuration conf;
   private final PrintStream out;
+  private final SimpleDateFormat isoDateFormat;
   private String[] stringTable;
 
+  public static SimpleDateFormat createSimpleDateFormat() {
+    SimpleDateFormat format =
+      new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS");
+    format.setTimeZone(TimeZone.getTimeZone("UTC"));
+    return format;
+  }
+
   public PBImageXmlWriter(Configuration conf, PrintStream out) {
     this.conf = conf;
     this.out = out;
+    this.isoDateFormat = createSimpleDateFormat();
   }
 
   public void visit(RandomAccessFile file) throws IOException {
@@ -80,6 +104,16 @@ public final class PBImageXmlWriter {
     try (FileInputStream fin = new FileInputStream(file.getFD())) {
       out.print("<?xml version=\"1.0\"?>\n<fsimage>");
 
+      out.print("<version>");
+      o("layoutVersion", summary.getLayoutVersion());
+      o("onDiskVersion", summary.getOndiskVersion());
+      // Output the version of OIV (which is not necessarily the version of
+      // the fsimage file).  This could be helpful in the case where a bug
+      // in OIV leads to information loss in the XML-- we can quickly tell
+      // if a specific fsimage XML file is affected by this bug.
+      o("oivRevision", VersionInfo.getRevision());
+      out.print("</version>\n");
+
       ArrayList<FileSummary.Section> sections = Lists.newArrayList(summary
           .getSectionsList());
       Collections.sort(sections, new Comparator<FileSummary.Section>() {
@@ -146,6 +180,8 @@ public final class PBImageXmlWriter {
     out.print("<CacheManagerSection>");
     CacheManagerSection s = CacheManagerSection.parseDelimitedFrom(is);
     o("nextDirectiveId", s.getNextDirectiveId());
+    o("numDirectives", s.getNumDirectives());
+    o("numPools", s.getNumPools());
     for (int i = 0; i < s.getNumPools(); ++i) {
       CachePoolInfoProto p = CachePoolInfoProto.parseDelimitedFrom(is);
       out.print("<pool>");
@@ -163,7 +199,7 @@ public final class PBImageXmlWriter {
           .o("replication", p.getReplication()).o("pool", p.getPool());
       out.print("<expiration>");
       CacheDirectiveInfoExpirationProto e = p.getExpiration();
-      o("millis", e.getMillis()).o("relatilve", e.getIsRelative());
+      o("millis", e.getMillis()).o("relative", e.getIsRelative());
       out.print("</expiration>\n");
       out.print("</directive>\n");
     }
@@ -187,13 +223,48 @@ public final class PBImageXmlWriter {
     out.print("</FileUnderConstructionSection>\n");
   }
 
+  private void dumpXattrs(INodeSection.XAttrFeatureProto xattrs) {
+    out.print("<xattrs>");
+    for (INodeSection.XAttrCompactProto xattr : xattrs.getXAttrsList()) {
+      out.print("<xattr>");
+      int encodedName = xattr.getName();
+      int ns = (XATTR_NAMESPACE_MASK & (encodedName >> XATTR_NAMESPACE_OFFSET)) |
+          ((XATTR_NAMESPACE_EXT_MASK & (encodedName >> XATTR_NAMESPACE_EXT_OFFSET)) << 2);
+      o("ns", XAttrProtos.XAttrProto.
+          XAttrNamespaceProto.valueOf(ns).toString());
+      o("name", stringTable[XATTR_NAME_MASK & (encodedName >> XATTR_NAME_OFFSET)]);
+      ByteString val = xattr.getValue();
+      if (val.isValidUtf8()) {
+        o("val", val.toStringUtf8());
+      } else {
+        o("valHex", Hex.encodeHexString(val.toByteArray()));
+      }
+      out.print("</xattr>");
+    }
+    out.print("</xattrs>");
+  }
+
   private void dumpINodeDirectory(INodeDirectory d) {
     o("mtime", d.getModificationTime()).o("permission",
         dumpPermission(d.getPermission()));
+    if (d.hasXAttrs()) {
+      dumpXattrs(d.getXAttrs());
+    }
     dumpAcls(d.getAcl());
     if (d.hasDsQuota() && d.hasNsQuota()) {
       o("nsquota", d.getNsQuota()).o("dsquota", d.getDsQuota());
     }
+    INodeSection.QuotaByStorageTypeFeatureProto typeQuotas =
+      d.getTypeQuotas();
+    if (typeQuotas != null) {
+      for (INodeSection.QuotaByStorageTypeEntryProto entry:
+            typeQuotas.getQuotasList()) {
+        out.print("<typeQuota>");
+        o("type", entry.getStorageType().toString());
+        o("quota", entry.getQuota());
+        out.print("</typeQuota>");
+      }
+    }
   }
 
   private void dumpINodeDirectorySection(InputStream in) throws IOException {
@@ -208,10 +279,10 @@ public final class PBImageXmlWriter {
       out.print("<directory>");
       o("parent", e.getParent());
       for (long id : e.getChildrenList()) {
-        o("inode", id);
+        o("child", id);
       }
       for (int refId : e.getRefChildrenList()) {
-        o("inodereference-index", refId);
+        o("refChild", refId);
       }
       out.print("</directory>\n");
     }
@@ -244,6 +315,9 @@ public final class PBImageXmlWriter {
         .o("atime", f.getAccessTime())
         .o("preferredBlockSize", f.getPreferredBlockSize())
         .o("permission", dumpPermission(f.getPermission()));
+    if (f.hasXAttrs()) {
+      dumpXattrs(f.getXAttrs());
+    }
     dumpAcls(f.getAcl());
     if (f.getBlocksCount() > 0) {
       out.print("<blocks>");
@@ -255,6 +329,12 @@ public final class PBImageXmlWriter {
       }
       out.print("</blocks>\n");
     }
+    if (f.hasStoragePolicyID()) {
+      o("storagePolicyId", f.getStoragePolicyID());
+    }
+    if (f.getIsStriped()) {
+      out.print("<isStriped/>");
+    }
 
     if (f.hasFileUC()) {
       INodeSection.FileUnderConstructionFeature u = f.getFileUC();
@@ -281,25 +361,28 @@ public final class PBImageXmlWriter {
     INodeSection s = INodeSection.parseDelimitedFrom(in);
     out.print("<INodeSection>");
     o("lastInodeId", s.getLastInodeId());
+    o("numInodes", s.getNumInodes());
     for (int i = 0; i < s.getNumInodes(); ++i) {
       INodeSection.INode p = INodeSection.INode.parseDelimitedFrom(in);
       out.print("<inode>");
-      o("id", p.getId()).o("type", p.getType()).o("name",
-          p.getName().toStringUtf8());
-
-      if (p.hasFile()) {
-        dumpINodeFile(p.getFile());
-      } else if (p.hasDirectory()) {
-        dumpINodeDirectory(p.getDirectory());
-      } else if (p.hasSymlink()) {
-        dumpINodeSymlink(p.getSymlink());
-      }
-
+      dumpINodeFields(p);
       out.print("</inode>\n");
     }
     out.print("</INodeSection>\n");
   }
 
+  private void dumpINodeFields(INodeSection.INode p) {
+    o("id", p.getId()).o("type", p.getType()).o("name",
+        p.getName().toStringUtf8());
+    if (p.hasFile()) {
+      dumpINodeFile(p.getFile());
+    } else if (p.hasDirectory()) {
+      dumpINodeDirectory(p.getDirectory());
+    } else if (p.hasSymlink()) {
+      dumpINodeSymlink(p.getSymlink());
+    }
+  }
+
   private void dumpINodeSymlink(INodeSymlink s) {
     o("permission", dumpPermission(s.getPermission()))
         .o("target", s.getTarget().toStringUtf8())
@@ -308,7 +391,8 @@ public final class PBImageXmlWriter {
 
   private void dumpNameSection(InputStream in) throws IOException {
     NameSystemSection s = NameSystemSection.parseDelimitedFrom(in);
-    out.print("<NameSection>\n");
+    out.print("<NameSection>");
+    o("namespaceId", s.getNamespaceId());
     o("genstampV1", s.getGenstampV1()).o("genstampV2", s.getGenstampV2())
         .o("genstampV1Limit", s.getGenstampV1Limit())
         .o("lastAllocatedBlockId", s.getLastAllocatedBlockId())
@@ -317,18 +401,73 @@ public final class PBImageXmlWriter {
   }
 
   private String dumpPermission(long permission) {
-    return FSImageFormatPBINode.Loader.loadPermission(permission, stringTable)
-        .toString();
+    PermissionStatus permStatus = FSImageFormatPBINode.Loader.
+        loadPermission(permission, stringTable);
+    return String.format("%s:%s:%04o", permStatus.getUserName(),
+        permStatus.getGroupName(), permStatus.getPermission().toExtendedShort());
   }
 
   private void dumpSecretManagerSection(InputStream is) throws IOException {
     out.print("<SecretManagerSection>");
     SecretManagerSection s = SecretManagerSection.parseDelimitedFrom(is);
+    int expectedNumDelegationKeys = s.getNumKeys();
+    int expectedNumTokens = s.getNumTokens();
     o("currentId", s.getCurrentId()).o("tokenSequenceNumber",
-        s.getTokenSequenceNumber());
+        s.getTokenSequenceNumber()).
+        o("numDelegationKeys", expectedNumDelegationKeys).
+        o("numTokens", expectedNumTokens);
+    for (int i = 0; i < expectedNumDelegationKeys; i++) {
+      SecretManagerSection.DelegationKey dkey =
+          SecretManagerSection.DelegationKey.parseDelimitedFrom(is);
+      out.print("<delegationKey>");
+      o("id", dkey.getId());
+      o("key", Hex.encodeHexString(dkey.getKey().toByteArray()));
+      if (dkey.hasExpiryDate()) {
+        dumpDate("expiry", dkey.getExpiryDate());
+      }
+      out.print("</delegationKey>");
+    }
+    for (int i = 0; i < expectedNumTokens; i++) {
+      SecretManagerSection.PersistToken token =
+          SecretManagerSection.PersistToken.parseDelimitedFrom(is);
+      out.print("<token>");
+      if (token.hasVersion()) {
+        o("version", token.getVersion());
+      }
+      if (token.hasOwner()) {
+        o("owner", token.getOwner());
+      }
+      if (token.hasRenewer()) {
+        o("renewer", token.getRenewer());
+      }
+      if (token.hasRealUser()) {
+        o("realUser", token.getRealUser());
+      }
+      if (token.hasIssueDate()) {
+        dumpDate("issueDate", token.getIssueDate());
+      }
+      if (token.hasMaxDate()) {
+        dumpDate("maxDate", token.getMaxDate());
+      }
+      if (token.hasSequenceNumber()) {
+        o("sequenceNumber", token.getSequenceNumber());
+      }
+      if (token.hasMasterKeyId()) {
+        o("masterKeyId", token.getMasterKeyId());
+      }
+      if (token.hasExpiryDate()) {
+        dumpDate("expiryDate", token.getExpiryDate());
+      }
+      out.print("</token>");
+    }
     out.print("</SecretManagerSection>");
   }
 
+  private void dumpDate(String tag, long date) {
+    out.print("<" + tag + ">" +
+      isoDateFormat.format(new Date(date)) + "</" + tag + ">");
+  }
+
   private void dumpSnapshotDiffSection(InputStream in) throws IOException {
     out.print("<SnapshotDiffSection>");
     while (true) {
@@ -337,30 +476,46 @@ public final class PBImageXmlWriter {
       if (e == null) {
         break;
       }
-      out.print("<diff>");
-      o("inodeid", e.getInodeId());
+      switch (e.getType()) {
+      case FILEDIFF:
+        out.print("<fileDiffEntry>");
+        break;
+      case DIRECTORYDIFF:
+        out.print("<dirDiffEntry>");
+        break;
+      default:
+        throw new IOException("unknown DiffEntry type " + e.getType());
+      }
+      o("inodeId", e.getInodeId());
+      o("count", e.getNumOfDiff());
       switch (e.getType()) {
       case FILEDIFF: {
         for (int i = 0; i < e.getNumOfDiff(); ++i) {
-          out.print("<filediff>");
+          out.print("<fileDiff>");
           SnapshotDiffSection.FileDiff f = SnapshotDiffSection.FileDiff
               .parseDelimitedFrom(in);
           o("snapshotId", f.getSnapshotId()).o("size", f.getFileSize()).o(
               "name", f.getName().toStringUtf8());
-          out.print("</filediff>\n");
+          out.print("</fileDiff>\n");
         }
       }
         break;
       case DIRECTORYDIFF: {
         for (int i = 0; i < e.getNumOfDiff(); ++i) {
-          out.print("<dirdiff>");
+          out.print("<dirDiff>");
           SnapshotDiffSection.DirectoryDiff d = SnapshotDiffSection.DirectoryDiff
               .parseDelimitedFrom(in);
           o("snapshotId", d.getSnapshotId())
-              .o("isSnapshotroot", d.getIsSnapshotRoot())
               .o("childrenSize", d.getChildrenSize())
-              .o("name", d.getName().toStringUtf8());
-
+              .o("isSnapshotRoot", d.getIsSnapshotRoot())
+              .o("name", d.getName().toStringUtf8())
+              .o("createdListSize", d.getCreatedListSize());
+          for (long did : d.getDeletedINodeList()) {
+            o("deletedInode", did);
+          }
+          for (int dRefid : d.getDeletedINodeRefList()) {
+            o("deletedInoderef", dRefid);
+          }
           for (int j = 0; j < d.getCreatedListSize(); ++j) {
             SnapshotDiffSection.CreatedListEntry ce = SnapshotDiffSection.CreatedListEntry
                 .parseDelimitedFrom(in);
@@ -368,24 +523,23 @@ public final class PBImageXmlWriter {
             o("name", ce.getName().toStringUtf8());
             out.print("</created>\n");
           }
-          for (long did : d.getDeletedINodeList()) {
-            out.print("<deleted>");
-            o("inode", did);
-            out.print("</deleted>\n");
-          }
-          for (int dRefid : d.getDeletedINodeRefList()) {
-            out.print("<deleted>");
-            o("inodereference-index", dRefid);
-            out.print("</deleted>\n");
-          }
-          out.print("</dirdiff>\n");
+          out.print("</dirDiff>\n");
         }
-      }
         break;
+      }
       default:
         break;
       }
-      out.print("</diff>");
+      switch (e.getType()) {
+      case FILEDIFF:
+        out.print("</fileDiffEntry>");
+        break;
+      case DIRECTORYDIFF:
+        out.print("</dirDiffEntry>");
+        break;
+      default:
+        throw new IOException("unknown DiffEntry type " + e.getType());
+      }
     }
     out.print("</SnapshotDiffSection>\n");
   }
@@ -394,6 +548,7 @@ public final class PBImageXmlWriter {
     out.print("<SnapshotSection>");
     SnapshotSection s = SnapshotSection.parseDelimitedFrom(in);
     o("snapshotCounter", s.getSnapshotCounter());
+    o("numSnapshots", s.getNumSnapshots());
     if (s.getSnapshottableDirCount() > 0) {
       out.print("<snapshottableDir>");
       for (long id : s.getSnapshottableDirList()) {
@@ -404,7 +559,12 @@ public final class PBImageXmlWriter {
     for (int i = 0; i < s.getNumSnapshots(); ++i) {
       SnapshotSection.Snapshot pbs = SnapshotSection.Snapshot
           .parseDelimitedFrom(in);
-      o("snapshot", pbs.getSnapshotId());
+      out.print("<snapshot>");
+      o("id", pbs.getSnapshotId());
+      out.print("<root>");
+      dumpINodeFields(pbs.getRoot());
+      out.print("</root>");
+      out.print("</snapshot>");
     }
     out.print("</SnapshotSection>\n");
   }
@@ -420,6 +580,14 @@ public final class PBImageXmlWriter {
   }
 
   private PBImageXmlWriter o(final String e, final Object v) {
+    if (v instanceof Boolean) {
+      // For booleans, the presence of the element indicates true, and its
+      // absence indicates false.
+      if ((Boolean)v != false) {
+        out.print("<" + e + "/>");
+      }
+      return this;
+    }
     out.print("<" + e + ">" +
         XMLUtils.mangleXmlString(v.toString(), true) + "</" + e + ">");
     return this;

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java

@@ -47,6 +47,8 @@ import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.parsers.SAXParser;
 import javax.xml.parsers.SAXParserFactory;
 
+import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.output.NullOutputStream;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -66,6 +68,8 @@ import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.log4j.Level;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -89,8 +93,7 @@ public class TestOfflineImageViewer {
   final static HashMap<String, FileStatus> writtenFiles = Maps.newHashMap();
   static int dirCount = 0;
 
-  @Rule
-  public TemporaryFolder folder = new TemporaryFolder();
+  private static File tempDir;
 
   // Create a populated namespace for later testing. Save its contents to a
   // data structure and store its fsimage location.
@@ -98,6 +101,7 @@ public class TestOfflineImageViewer {
   // multiple tests.
   @BeforeClass
   public static void createOriginalFSImage() throws IOException {
+    tempDir = Files.createTempDir();
     MiniDFSCluster cluster = null;
     try {
       Configuration conf = new Configuration();
@@ -169,6 +173,9 @@ public class TestOfflineImageViewer {
       hdfs.setXAttr(xattr, "user.a2", new byte[]{ 0x37, 0x38, 0x39 });
       // OIV should be able to handle empty value XAttrs
       hdfs.setXAttr(xattr, "user.a3", null);
+      // OIV should be able to handle XAttr values that can't be expressed
+      // as UTF8
+      hdfs.setXAttr(xattr, "user.a4", new byte[]{ -0x3d, 0x28 });
       writtenFiles.put(xattr.toString(), hdfs.getFileStatus(xattr));
 
       // Write results to the fsimage file
@@ -190,6 +197,7 @@ public class TestOfflineImageViewer {
 
   @AfterClass
   public static void deleteOriginalFSImage() throws IOException {
+    FileUtils.deleteQuietly(tempDir);
     if (originalFsimage != null && originalFsimage.exists()) {
       originalFsimage.delete();
     }
@@ -204,7 +212,7 @@ public class TestOfflineImageViewer {
 
   @Test(expected = IOException.class)
   public void testTruncatedFSImage() throws IOException {
-    File truncatedFile = folder.newFile();
+    File truncatedFile = new File(tempDir, "truncatedFsImage");
     PrintStream output = new PrintStream(NullOutputStream.NULL_OUTPUT_STREAM);
     copyPartOfFile(originalFsimage, truncatedFile);
     new FileDistributionCalculator(new Configuration(), 0, 0, output)
@@ -449,4 +457,46 @@ public class TestOfflineImageViewer {
     connection.connect();
     assertEquals(expectedCode, connection.getResponseCode());
   }
+
+  /**
+   * Tests the ReverseXML processor.
+   *
+   * 1. Translate fsimage -> reverseImage.xml
+   * 2. Translate reverseImage.xml -> reverseImage
+   * 3. Translate reverseImage -> reverse2Image.xml
+   * 4. Verify that reverseImage.xml and reverse2Image.xml match
+   *
+   * @throws Throwable
+   */
+  @Test
+  public void testReverseXmlRoundTrip() throws Throwable {
+    GenericTestUtils.setLogLevel(OfflineImageReconstructor.LOG,
+        Level.TRACE);
+    File reverseImageXml = new File(tempDir, "reverseImage.xml");
+    File reverseImage =  new File(tempDir, "reverseImage");
+    File reverseImage2Xml =  new File(tempDir, "reverseImage2.xml");
+    LOG.info("Creating reverseImage.xml=" + reverseImageXml.getAbsolutePath() +
+        ", reverseImage=" + reverseImage.getAbsolutePath() +
+        ", reverseImage2Xml=" + reverseImage2Xml.getAbsolutePath());
+    if (OfflineImageViewerPB.run(new String[] { "-p", "XML",
+         "-i", originalFsimage.getAbsolutePath(),
+         "-o", reverseImageXml.getAbsolutePath() }) != 0) {
+      throw new IOException("oiv returned failure creating first XML file.");
+    }
+    if (OfflineImageViewerPB.run(new String[] { "-p", "ReverseXML",
+          "-i", reverseImageXml.getAbsolutePath(),
+          "-o", reverseImage.getAbsolutePath() }) != 0) {
+      throw new IOException("oiv returned failure recreating fsimage file.");
+    }
+    if (OfflineImageViewerPB.run(new String[] { "-p", "XML",
+        "-i", reverseImage.getAbsolutePath(),
+        "-o", reverseImage2Xml.getAbsolutePath() }) != 0) {
+      throw new IOException("oiv returned failure creating second " +
+          "XML file.");
+    }
+    // The XML file we wrote based on the re-created fsimage should be the
+    // same as the one we dumped from the original fsimage.
+    Assert.assertEquals("",
+      GenericTestUtils.getFilesDiff(reverseImageXml, reverseImage2Xml));
+  }
 }