HDFS-9229. Expose size of NameNode directory as a metric. Contributed by Surendra Singh Lilhore.

Change-Id: I985627a5d1400249d72d084283ef366d5ac6e07b

hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md

@@ -240,6 +240,7 @@ Each metrics record contains tags such as HAState and Hostname as additional inf
 | `LockQueueLength` | Number of threads waiting to acquire FSNameSystem lock |
 | `TotalSyncCount` | Total number of sync operations performed by edit log |
 | `TotalSyncTimes` | Total number of milliseconds spent by various edit logs in sync operation|
+| `NameDirSize` | NameNode name directories size in bytes |
 
 JournalNode
 -----------

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -1608,6 +1608,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-8545. Refactor FS#getUsed() to use ContentSummary and add an API to fetch
     the total file length from a specific path (J.Andreina via vinayakumarb)
 
+    HDFS-9229. Expose size of NameNode directory as a metric.
+    (Surendra Singh Lilhore via zhz)
+
   OPTIMIZATIONS
 
     HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java

@@ -30,6 +30,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Properties;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -310,6 +311,20 @@ public abstract class Storage extends StorageInfo {
       return dirType;
     }    
 
+    /**
+     * Get storage directory size.
+     */
+    public long getDirecorySize() {
+      try {
+        if (!isShared() && root != null && root.exists()) {
+          return FileUtils.sizeOfDirectory(root);
+        }
+      } catch (Exception e) {
+        LOG.warn("Failed to get directory size :" + root, e);
+      }
+      return 0;
+    }
+
     public void read(File from, Storage storage) throws IOException {
       Properties props = readPropertiesFile(from);
       storage.setFieldsFromProperties(props, this);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java

@@ -1064,6 +1064,8 @@ public class FSImage implements Closeable {
     } finally {
       removeFromCheckpointing(imageTxId);
     }
+    //Update NameDirSize Metric
+    getStorage().updateNameDirSize();
   }
 
   /**
@@ -1244,6 +1246,8 @@ public class FSImage implements Closeable {
     // we won't miss this log segment on a restart if the edits directories
     // go missing.
     storage.writeTransactionIdFileToStorage(getEditLog().getCurSegmentTxId());
+    //Update NameDirSize Metric
+    getStorage().updateNameDirSize();
     return new CheckpointSignature(this);
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -6402,6 +6402,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
     return VersionInfo.getVersion();
   }
 
+  @Override // NameNodeStatusMXBean
+  public String getNameDirSize() {
+    return getFSImage().getStorage().getNNDirectorySize();
+  }
+
   /**
    * Verifies that the given identifier and password are valid and match.
    * @param identifier Token identifier.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java

@@ -29,6 +29,7 @@ import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 import java.util.UUID;
 import java.util.concurrent.CopyOnWriteArrayList;
@@ -52,6 +53,7 @@ import org.apache.hadoop.hdfs.util.PersistentLongFile;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.net.DNS;
 import org.apache.hadoop.util.Time;
+import org.mortbay.util.ajax.JSON;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
@@ -148,6 +150,11 @@ public class NNStorage extends Storage implements Closeable,
    */
   private HashMap<String, String> deprecatedProperties;
 
+  /**
+   * Name directories size for metric.
+   */
+  private Map<String, Long> nameDirSizeMap = new HashMap<>();
+
   /**
    * Construct the NNStorage.
    * @param conf Namenode configuration.
@@ -166,6 +173,8 @@ public class NNStorage extends Storage implements Closeable,
     setStorageDirectories(imageDirs, 
                           Lists.newArrayList(editsDirs),
                           FSNamesystem.getSharedEditsDirs(conf));
+    //Update NameDirSize metric value after NN start
+    updateNameDirSize();
   }
 
   @Override // Storage
@@ -1075,4 +1084,20 @@ public class NNStorage extends Storage implements Closeable,
         getBlockPoolID(),
         getCTime());
   }
+
+  public String getNNDirectorySize() {
+    return JSON.toString(nameDirSizeMap);
+  }
+
+  public void updateNameDirSize() {
+    Map<String, Long> nnDirSizeMap = new HashMap<>();
+    for (Iterator<StorageDirectory> it = dirIterator(); it.hasNext();) {
+      StorageDirectory sd = it.next();
+      if (!sd.isShared()) {
+        nnDirSizeMap.put(sd.getRoot().getAbsolutePath(), sd.getDirecorySize());
+      }
+    }
+    nameDirSizeMap.clear();
+    nameDirSizeMap.putAll(nnDirSizeMap);
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java

@@ -272,4 +272,9 @@ public interface NameNodeMXBean {
    */
   public Map<String, Integer> getDistinctVersions();
   
+  /**
+   * Get namenode directory size.
+   */
+  String getNameDirSize();
+
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java

@@ -372,6 +372,8 @@ public class EditLogTailer {
           } finally {
             namesystem.cpUnlock();
           }
+          //Update NameDirSize Metric
+          namesystem.getFSImage().getStorage().updateNameDirSize();
         } catch (EditLogInputException elie) {
           LOG.warn("Error while reading edits from disk. Will try again.", elie);
         } catch (InterruptedException ie) {
@@ -463,4 +465,4 @@ public class EditLogTailer {
       return cachedActiveProxy;
     }
   }
-}
+}

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java

@@ -26,12 +26,16 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
 import org.apache.hadoop.hdfs.server.namenode.top.TopConf;
 import org.apache.hadoop.io.nativeio.NativeIO;
 import org.apache.hadoop.io.nativeio.NativeIO.POSIX.NoMlockCacheManipulator;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.apache.hadoop.util.VersionInfo;
 import org.codehaus.jackson.map.ObjectMapper;
 import org.junit.Test;
@@ -40,6 +44,7 @@ import org.mortbay.util.ajax.JSON;
 import javax.management.MBeanServer;
 import javax.management.ObjectName;
 import java.io.File;
+import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.net.URI;
 import java.util.Collection;
@@ -186,7 +191,7 @@ public class TestNameNodeMXBean {
       }
       assertEquals(2, statusMap.get("active").size());
       assertEquals(0, statusMap.get("failed").size());
-      
+
       // This will cause the first dir to fail.
       File failedNameDir = new File(nameDirUris.iterator().next());
       assertEquals(0, FileUtil.chmod(
@@ -412,4 +417,59 @@ public class TestNameNodeMXBean {
       }
     }
   }
+
+  @Test(timeout = 120000)
+  public void testNNDirectorySize() throws Exception{
+    Configuration conf = new Configuration();
+    conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+    // Have to specify IPC ports so the NNs can talk to each other.
+    MiniDFSNNTopology topology = new MiniDFSNNTopology()
+        .addNameservice(new MiniDFSNNTopology.NSConf("ns1")
+            .addNN(new MiniDFSNNTopology.NNConf("nn1")
+                .setIpcPort(ServerSocketUtil.getPort(0, 100)))
+            .addNN(new MiniDFSNNTopology.NNConf("nn2")
+                .setIpcPort(ServerSocketUtil.getPort(0, 100))));
+
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+        .nnTopology(topology).numDataNodes(0)
+        .build();
+    FileSystem fs = null;
+    try {
+      cluster.waitActive();
+
+      FSNamesystem nn0 = cluster.getNamesystem(0);
+      FSNamesystem nn1 = cluster.getNamesystem(1);
+      checkNNDirSize(cluster.getNameDirs(0), nn0.getNameDirSize());
+      checkNNDirSize(cluster.getNameDirs(1), nn1.getNameDirSize());
+      cluster.transitionToActive(0);
+      fs = cluster.getFileSystem(0);
+      DFSTestUtil.createFile(fs, new Path("/file"), 0, (short) 1, 0L);
+
+      //rollEditLog
+      HATestUtil.waitForStandbyToCatchUp(cluster.getNameNode(0),
+          cluster.getNameNode(1));
+      checkNNDirSize(cluster.getNameDirs(0), nn0.getNameDirSize());
+      checkNNDirSize(cluster.getNameDirs(1), nn1.getNameDirSize());
+
+      //Test metric after call saveNamespace
+      DFSTestUtil.createFile(fs, new Path("/file"), 0, (short) 1, 0L);
+      nn0.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+      nn0.saveNamespace(0, 0);
+      checkNNDirSize(cluster.getNameDirs(0), nn0.getNameDirSize());
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  private void checkNNDirSize(Collection<URI> nameDirUris, String metric){
+    Map<String, Long> nnDirMap =
+        (Map<String, Long>) JSON.parse(metric);
+    assertEquals(nameDirUris.size(), nnDirMap.size());
+    for (URI dirUrl : nameDirUris) {
+      File dir = new File(dirUrl);
+      assertEquals(nnDirMap.get(dir.getAbsolutePath()).longValue(),
+          FileUtils.sizeOfDirectory(dir));
+    }
+  }
 }