HADOOP-3683. Fix dfs metrics to count file listings rather than files

listed. Contributed by lohit vijayarenu.


git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@673893 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -751,6 +751,9 @@ Release 0.18.0 - Unreleased
     HADOOP-3678. Avoid spurious exceptions logged at DataNode when clients
     read from DFS. (rangadi)
 
+    HADOOP-3683. Fix dfs metrics to count file listings rather than files
+    listed. (lohit vijayarenu via cdouglas)
+
 Release 0.17.1 - Unreleased
 
   INCOMPATIBLE CHANGES

src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java

@@ -296,6 +296,7 @@ public class NameNode implements ClientProtocol, DatanodeProtocol,
             null, masked),
         clientName, clientMachine, overwrite, replication, blockSize);
     myMetrics.numFilesCreated.inc();
+    myMetrics.numCreateFileOps.inc();
   }
 
   /** Coming in a future release.... */
@@ -336,7 +337,10 @@ public class NameNode implements ClientProtocol, DatanodeProtocol,
                                String clientName) throws IOException {
     stateChangeLog.debug("*BLOCK* NameNode.addBlock: file "
                          +src+" for "+clientName);
-    return namesystem.getAdditionalBlock(src, clientName);
+    LocatedBlock locatedBlock = namesystem.getAdditionalBlock(src, clientName);
+    if (locatedBlock != null)
+      myMetrics.numAddBlockOps.inc();
+    return locatedBlock;
   }
 
   /**
@@ -428,7 +432,10 @@ public class NameNode implements ClientProtocol, DatanodeProtocol,
       stateChangeLog.debug("*DIR* Namenode.delete: src=" + src
           + ", recursive=" + recursive);
     }
-    return namesystem.delete(src, recursive);
+    boolean ret = namesystem.delete(src, recursive);
+    if (ret) 
+      myMetrics.numDeleteFileOps.inc();
+    return ret;
   }
 
   /**
@@ -466,7 +473,7 @@ public class NameNode implements ClientProtocol, DatanodeProtocol,
   public DFSFileInfo[] getListing(String src) throws IOException {
     DFSFileInfo[] files = namesystem.getListing(src);
     if (files != null) {
-      myMetrics.numFilesListed.inc(files.length);
+      myMetrics.numGetListingOps.inc();
     }
     return files;
   }

src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java

@@ -49,9 +49,15 @@ public class NameNodeMetrics implements Updater {
     public MetricsTimeVaryingInt numFilesCreated = new MetricsTimeVaryingInt("FilesCreated");
     public MetricsTimeVaryingInt numGetBlockLocations = new MetricsTimeVaryingInt("GetBlockLocations");
     public MetricsTimeVaryingInt numFilesRenamed = new MetricsTimeVaryingInt("FilesRenamed");
-    public MetricsTimeVaryingInt numFilesListed = new MetricsTimeVaryingInt("FilesListed");
+    public MetricsTimeVaryingInt numGetListingOps = 
+                                   new MetricsTimeVaryingInt("GetListingOps");
+    public MetricsTimeVaryingInt numCreateFileOps = 
+                                   new MetricsTimeVaryingInt("CreateFileOps");
+    public MetricsTimeVaryingInt numDeleteFileOps = 
+                                   new MetricsTimeVaryingInt("DeleteFileOps");
+    public MetricsTimeVaryingInt numAddBlockOps = 
+                                   new MetricsTimeVaryingInt("AddBlockOps");
 
-    
     public MetricsTimeVaryingRate transactions = new MetricsTimeVaryingRate("Transactions");
     public MetricsTimeVaryingRate syncs = new MetricsTimeVaryingRate("Syncs");
     public MetricsTimeVaryingRate blockReport = new MetricsTimeVaryingRate("blockReport");
@@ -95,8 +101,10 @@ public class NameNodeMetrics implements Updater {
         numFilesCreated.pushMetric(metricsRecord);
         numGetBlockLocations.pushMetric(metricsRecord);
         numFilesRenamed.pushMetric(metricsRecord);
-        numFilesListed.pushMetric(metricsRecord);
-
+        numGetListingOps.pushMetric(metricsRecord);
+        numCreateFileOps.pushMetric(metricsRecord);
+        numDeleteFileOps.pushMetric(metricsRecord);
+        numAddBlockOps.pushMetric(metricsRecord);
 
         transactions.pushMetric(metricsRecord);
         syncs.pushMetric(metricsRecord);

src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeStatistics.java

@@ -163,11 +163,40 @@ public class NameNodeStatistics implements NameNodeStatisticsMBean {
     return myMetrics.numFilesCreated.getPreviousIntervalValue();
   }
 
+  /** 
+   *@deprecated call getNumGetListingOps() instead
+   */
+  @Deprecated
+  public int getNumFilesListed() {
+    return getNumGetListingOps();
+  }
+
   /**
    * @inheritDoc
    */
-  public int getNumFilesListed() {
-    return myMetrics.numFilesListed.getPreviousIntervalValue();
+  public int getNumGetListingOps() {
+    return myMetrics.numGetListingOps.getPreviousIntervalValue();
+  }
+
+  /**
+   * @inheritDoc
+   */
+  public int getNumCreateFileOps() {
+    return myMetrics.numCreateFileOps.getPreviousIntervalValue();
+  }
+
+  /**
+   * @inheritDoc
+   */
+  public int getNumDeleteFileOps() {
+    return myMetrics.numDeleteFileOps.getPreviousIntervalValue();
+  }
+
+  /**
+   * @inheritDoc
+   */
+  public int getNumAddBlockOps() {
+    return myMetrics.numAddBlockOps.getPreviousIntervalValue();
   }
 
   /** @inheritDoc */

src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeStatisticsMBean.java

@@ -156,6 +156,32 @@ public interface NameNodeStatisticsMBean {
   /**
    *   Number of files listed in the last interval
    * @return number of operations
+   * @deprecated Use getNumGetListingOps() instead
    */
+  @Deprecated
   int getNumFilesListed();
+
+  /**
+   *   Number of files listed in the last interval
+   * @return number of operations
+   */
+  int getNumGetListingOps();
+
+  /**
+   *   Number of file creation operations in the last interval
+   * @return number of file creation operations
+   */
+  int getNumCreateFileOps();
+
+  /**
+   *   Number of file deletion operations in the last interval
+   * @return number of file deletion operations
+   */
+  int getNumDeleteFileOps();
+
+  /**
+   *   Number of add block operations in the last interval
+   * @return number of add block operations
+   */
+  int getNumAddBlockOps();
 }