HDFS-9721. Allow Delimited PB OIV tool to run upon fsimage that contains INodeReference. (Xiao Chen via lei)

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -1872,6 +1872,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-9638. Improve DistCp Help and documentation.
     (Wei-Chiu Chuang via Yongjun Zhang)
 
+    HDFS-9721. Allow Delimited PB OIV tool to run upon fsimage that contains
+    INodeReference. (Xiao Chen via lei)
+
   OPTIMIZATIONS
 
     HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java

@@ -204,7 +204,7 @@ class FSImageLoader {
     return dirs;
   }
 
-  private static ImmutableList<Long> loadINodeReferenceSection(InputStream in)
+  static ImmutableList<Long> loadINodeReferenceSection(InputStream in)
       throws IOException {
     LOG.info("Loading inode references");
     ImmutableList.Builder<Long> builder = ImmutableList.builder();

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IgnoreSnapshotException.java

@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.IOException;
+
+/**
+ * Signals that a snapshot is ignored.
+ */
+public class IgnoreSnapshotException extends IOException {
+  public IgnoreSnapshotException() {
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java

@@ -84,6 +84,7 @@ public class PBImageDelimitedTextWriter extends PBImageTextWriter {
       inodeName.isEmpty() ? "/" : inodeName);
     buffer.append(path.toString());
     PermissionStatus p = null;
+    boolean isDir = false;
 
     switch (inode.getType()) {
     case FILE:
@@ -109,6 +110,7 @@ public class PBImageDelimitedTextWriter extends PBImageTextWriter {
       append(buffer, 0);  // Num bytes.
       append(buffer, dir.getNsQuota());
       append(buffer, dir.getDsQuota());
+      isDir = true;
       break;
     case SYMLINK:
       INodeSymlink s = inode.getSymlink();
@@ -126,9 +128,28 @@ public class PBImageDelimitedTextWriter extends PBImageTextWriter {
       break;
     }
     assert p != null;
-    append(buffer, p.getPermission().toString());
+    String dirString = isDir ? "d" : "-";
+    append(buffer, dirString + p.getPermission().toString());
     append(buffer, p.getUserName());
     append(buffer, p.getGroupName());
     return buffer.toString();
   }
+
+  @Override
+  public String getHeader() {
+    StringBuffer buffer = new StringBuffer();
+    buffer.append("Path");
+    append(buffer, "Replication");
+    append(buffer, "ModificationTime");
+    append(buffer, "AccessTime");
+    append(buffer, "PreferredBlockSize");
+    append(buffer, "BlocksCount");
+    append(buffer, "FileSize");
+    append(buffer, "NSQUOTA");
+    append(buffer, "DSQUOTA");
+    append(buffer, "Permission");
+    append(buffer, "UserName");
+    append(buffer, "GroupName");
+    return buffer.toString();
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.hdfs.tools.offlineImageViewer;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
@@ -195,13 +196,17 @@ abstract class PBImageTextWriter implements Closeable {
       dirMap.put(p.getId(), dir);
     }
 
+    @Override
     public String getParentPath(long inode) throws IOException {
       if (inode == INodeId.ROOT_INODE_ID) {
         return "";
       }
       Dir parent = dirChildMap.get(inode);
-      Preconditions.checkState(parent != null,
-          "Can not find parent directory for INode: %s", inode);
+      if (parent == null) {
+        // The inode is an INodeReference, which is generated from snapshot.
+        // For delimited oiv tool, no need to print out metadata in snapshots.
+        PBImageTextWriter.ignoreSnapshotName(inode);
+      }
       return parent.getPath();
     }
 
@@ -353,16 +358,22 @@ abstract class PBImageTextWriter implements Closeable {
         return "/";
       }
       byte[] bytes = dirChildMap.get(toBytes(inode));
-      Preconditions.checkState(bytes != null && bytes.length == 8,
-          "Can not find parent directory for inode %s, "
-              + "fsimage might be corrupted", inode);
+      if (bytes == null) {
+        // The inode is an INodeReference, which is generated from snapshot.
+        // For delimited oiv tool, no need to print out metadata in snapshots.
+        PBImageTextWriter.ignoreSnapshotName(inode);
+      }
+      if (bytes.length != 8) {
+        throw new IOException(
+            "bytes array length error. Actual length is " + bytes.length);
+      }
       long parent = toLong(bytes);
       if (!dirPathCache.containsKey(parent)) {
         bytes = dirMap.get(toBytes(parent));
-        if (parent != INodeId.ROOT_INODE_ID) {
-          Preconditions.checkState(bytes != null,
-              "Can not find parent directory for inode %s, "
-                  + ", the fsimage might be corrupted.", parent);
+        if (parent != INodeId.ROOT_INODE_ID && bytes == null) {
+          // The parent is an INodeReference, which is generated from snapshot.
+          // For delimited oiv tool, no need to print out metadata in snapshots.
+          PBImageTextWriter.ignoreSnapshotName(parent);
         }
         String parentName = toString(bytes);
         String parentPath =
@@ -401,6 +412,7 @@ abstract class PBImageTextWriter implements Closeable {
 
   @Override
   public void close() throws IOException {
+    out.flush();
     IOUtils.cleanup(null, metadataMap);
   }
 
@@ -411,6 +423,11 @@ abstract class PBImageTextWriter implements Closeable {
    */
   abstract protected String getEntry(String parent, INode inode);
 
+  /**
+   * Get text output for the header line.
+   */
+  abstract protected String getHeader();
+
   public void visit(RandomAccessFile file) throws IOException {
     Configuration conf = new Configuration();
     if (!FSImageUtil.checkFileFormat(file)) {
@@ -442,6 +459,7 @@ abstract class PBImageTextWriter implements Closeable {
             }
           });
 
+      ImmutableList<Long> refIdList = null;
       for (FileSummary.Section section : sections) {
         fin.getChannel().position(section.getOffset());
         is = FSImageUtil.wrapInputStreamForCompression(conf,
@@ -449,15 +467,22 @@ abstract class PBImageTextWriter implements Closeable {
                 fin, section.getLength())));
         switch (SectionName.fromString(section.getName())) {
         case STRING_TABLE:
+          LOG.info("Loading string table");
           stringTable = FSImageLoader.loadStringTable(is);
           break;
+        case INODE_REFERENCE:
+          // Load INodeReference so that all INodes can be processed.
+          // Snapshots are not handled and will just be ignored for now.
+          LOG.info("Loading inode references");
+          refIdList = FSImageLoader.loadINodeReferenceSection(is);
+          break;
         default:
           break;
         }
       }
 
       loadDirectories(fin, sections, summary, conf);
-      loadINodeDirSection(fin, sections, summary, conf);
+      loadINodeDirSection(fin, sections, summary, conf, refIdList);
       metadataMap.sync();
       output(conf, summary, fin, sections);
     }
@@ -468,6 +493,7 @@ abstract class PBImageTextWriter implements Closeable {
       throws IOException {
     InputStream is;
     long startTime = Time.monotonicNow();
+    out.println(getHeader());
     for (FileSummary.Section section : sections) {
       if (SectionName.fromString(section.getName()) == SectionName.INODE) {
         fin.getChannel().position(section.getOffset());
@@ -508,7 +534,7 @@ abstract class PBImageTextWriter implements Closeable {
 
   private void loadINodeDirSection(
       FileInputStream fin, List<FileSummary.Section> sections,
-      FileSummary summary, Configuration conf)
+      FileSummary summary, Configuration conf, List<Long> refIdList)
       throws IOException {
     LOG.info("Loading INode directory section.");
     long startTime = Time.monotonicNow();
@@ -519,7 +545,7 @@ abstract class PBImageTextWriter implements Closeable {
         InputStream is = FSImageUtil.wrapInputStreamForCompression(conf,
             summary.getCodec(), new BufferedInputStream(
                 new LimitInputStream(fin, section.getLength())));
-        buildNamespace(is);
+        buildNamespace(is, refIdList);
       }
     }
     long timeTaken = Time.monotonicNow() - startTime;
@@ -549,7 +575,8 @@ abstract class PBImageTextWriter implements Closeable {
   /**
    * Scan the INodeDirectory section to construct the namespace.
    */
-  private void buildNamespace(InputStream in) throws IOException {
+  private void buildNamespace(InputStream in, List<Long> refIdList)
+      throws IOException {
     int count = 0;
     while (true) {
       FsImageProto.INodeDirectorySection.DirEntry e =
@@ -562,12 +589,15 @@ abstract class PBImageTextWriter implements Closeable {
         LOG.debug("Scanned {} directories.", count);
       }
       long parentId = e.getParent();
-      // Referred INode is not support for now.
       for (int i = 0; i < e.getChildrenCount(); i++) {
         long childId = e.getChildren(i);
         metadataMap.putDirChild(parentId, childId);
       }
-      Preconditions.checkState(e.getRefChildrenCount() == 0);
+      for (int i = e.getChildrenCount();
+           i < e.getChildrenCount() + e.getRefChildrenCount(); i++) {
+        int refId = e.getRefChildren(i - e.getChildrenCount());
+        metadataMap.putDirChild(parentId, refIdList.get(refId));
+      }
     }
     LOG.info("Scanned {} INode directories to build namespace.", count);
   }
@@ -575,15 +605,41 @@ abstract class PBImageTextWriter implements Closeable {
   private void outputINodes(InputStream in) throws IOException {
     INodeSection s = INodeSection.parseDelimitedFrom(in);
     LOG.info("Found {} INodes in the INode section", s.getNumInodes());
+    long ignored = 0;
+    long ignoredSnapshots = 0;
     for (int i = 0; i < s.getNumInodes(); ++i) {
       INode p = INode.parseDelimitedFrom(in);
-      String parentPath = metadataMap.getParentPath(p.getId());
-      out.println(getEntry(parentPath, p));
+      try {
+        String parentPath = metadataMap.getParentPath(p.getId());
+        out.println(getEntry(parentPath, p));
+      } catch (IOException ioe) {
+        ignored++;
+        if (!(ioe instanceof IgnoreSnapshotException)) {
+          LOG.warn("Exception caught, ignoring node:{}", p.getId(), ioe);
+        } else {
+          ignoredSnapshots++;
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Exception caught, ignoring node:{}.", p.getId(), ioe);
+          }
+        }
+      }
 
       if (LOG.isDebugEnabled() && i % 100000 == 0) {
         LOG.debug("Outputted {} INodes.", i);
       }
     }
+    if (ignored > 0) {
+      LOG.warn("Ignored {} nodes, including {} in snapshots. Please turn on"
+              + " debug log for details", ignored, ignoredSnapshots);
+    }
     LOG.info("Outputted {} INodes.", s.getNumInodes());
   }
+
+  static void ignoreSnapshotName(long inode) throws IOException {
+    // Ignore snapshots - we want the output similar to -ls -R.
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("No snapshot name found for inode {}", inode);
+    }
+    throw new IgnoreSnapshotException();
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java

@@ -87,6 +87,7 @@ public class TestOfflineImageViewer {
 
   // namespace as written to dfs, to be compared with viewer's output
   final static HashMap<String, FileStatus> writtenFiles = Maps.newHashMap();
+  static int dirCount = 0;
 
   @Rule
   public TemporaryFolder folder = new TemporaryFolder();
@@ -113,7 +114,7 @@ public class TestOfflineImageViewer {
       DistributedFileSystem hdfs = cluster.getFileSystem();
 
       // Create a reasonable namespace
-      for (int i = 0; i < NUM_DIRS; i++) {
+      for (int i = 0; i < NUM_DIRS; i++, dirCount++) {
         Path dir = new Path("/dir" + i);
         hdfs.mkdirs(dir);
         writtenFiles.put(dir.toString(), pathToFileEntry(hdfs, dir.toString()));
@@ -131,11 +132,13 @@ public class TestOfflineImageViewer {
       // Create an empty directory
       Path emptydir = new Path("/emptydir");
       hdfs.mkdirs(emptydir);
+      dirCount++;
       writtenFiles.put(emptydir.toString(), hdfs.getFileStatus(emptydir));
 
       //Create a directory whose name should be escaped in XML
       Path invalidXMLDir = new Path("/dirContainingInvalidXMLChar\u0000here");
       hdfs.mkdirs(invalidXMLDir);
+      dirCount++;
 
       // Get delegation tokens so we log the delegation token op
       Token<?>[] delegationTokens = hdfs
@@ -144,15 +147,24 @@ public class TestOfflineImageViewer {
         LOG.debug("got token " + t);
       }
 
-      final Path snapshot = new Path("/snapshot");
-      hdfs.mkdirs(snapshot);
-      hdfs.allowSnapshot(snapshot);
-      hdfs.mkdirs(new Path("/snapshot/1"));
-      hdfs.delete(snapshot, true);
+      // Create INodeReference
+      final Path src = new Path("/src");
+      hdfs.mkdirs(src);
+      dirCount++;
+      writtenFiles.put(src.toString(), hdfs.getFileStatus(src));
+      final Path orig = new Path("/src/orig");
+      hdfs.mkdirs(orig);
+      hdfs.allowSnapshot(src);
+      hdfs.createSnapshot(src, "snapshot");
+      final Path dst = new Path("/dst");
+      hdfs.rename(orig, dst);
+      dirCount++;
+      writtenFiles.put(dst.toString(), hdfs.getFileStatus(dst));
 
       // Set XAttrs so the fsimage contains XAttr ops
       final Path xattr = new Path("/xattr");
       hdfs.mkdirs(xattr);
+      dirCount++;
       hdfs.setXAttr(xattr, "user.a1", new byte[]{ 0x31, 0x32, 0x33 });
       hdfs.setXAttr(xattr, "user.a2", new byte[]{ 0x37, 0x38, 0x39 });
       // OIV should be able to handle empty value XAttrs
@@ -232,8 +244,8 @@ public class TestOfflineImageViewer {
     matcher = p.matcher(outputString);
     assertTrue(matcher.find() && matcher.groupCount() == 1);
     int totalDirs = Integer.parseInt(matcher.group(1));
-    // totalDirs includes root directory, empty directory, and xattr directory
-    assertEquals(NUM_DIRS + 4, totalDirs);
+    // totalDirs includes root directory
+    assertEquals(dirCount + 1, totalDirs);
 
     FileStatus maxFile = Collections.max(writtenFiles.values(),
         new Comparator<FileStatus>() {
@@ -285,7 +297,7 @@ public class TestOfflineImageViewer {
 
       // verify the number of directories
       FileStatus[] statuses = webhdfs.listStatus(new Path("/"));
-      assertEquals(NUM_DIRS + 3, statuses.length); // contains empty and xattr directory
+      assertEquals(dirCount, statuses.length);
 
       // verify the number of files in the directory
       statuses = webhdfs.listStatus(new Path("/dir0"));
@@ -393,11 +405,15 @@ public class TestOfflineImageViewer {
         BufferedReader reader =
             new BufferedReader(new InputStreamReader(input))) {
       String line;
+      boolean header = true;
       while ((line = reader.readLine()) != null) {
         System.out.println(line);
         String[] fields = line.split(DELIMITER);
         assertEquals(12, fields.length);
-        fileNames.add(fields[0]);
+        if (!header) {
+          fileNames.add(fields[0]);
+        }
+        header = false;
       }
     }