HADOOP-13305. Define common statistics names across schemes. Contributed by Mingliang Liu.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java

@@ -106,6 +106,11 @@ public class FileSystemStorageStatistics extends StorageStatistics {
     this.stats = stats;
   }
 
+  @Override
+  public String getScheme() {
+    return stats.getScheme();
+  }
+
   @Override
   public Iterator<LongStatistic> getLongStatistics() {
     return new LongStatisticIterator(stats.getData());

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java

@@ -27,6 +27,51 @@ import java.util.Iterator;
  */
 @InterfaceAudience.Public
 public abstract class StorageStatistics {
+
+  /**
+   * These are common statistic names.
+   *
+   * The following names are considered general and preserved across different
+   * StorageStatistics classes. When implementing a new StorageStatistics, it is
+   * highly recommended to use the common statistic names.
+   *
+   * When adding new common statistic name constants, please make them unique.
+   * By convention, they are implicitly unique:
+   *  - the name of the constants are uppercase, words separated by underscores.
+   *  - the value of the constants are lowercase of the constant names.
+   */
+  public interface CommonStatisticNames {
+    // The following names are for file system operation invocations
+    String OP_APPEND = "op_append";
+    String OP_COPY_FROM_LOCAL_FILE = "op_copy_from_local_file";
+    String OP_CREATE = "op_create";
+    String OP_CREATE_NON_RECURSIVE = "op_create_non_recursive";
+    String OP_DELETE = "op_delete";
+    String OP_EXISTS = "op_exists";
+    String OP_GET_CONTENT_SUMMARY = "op_get_content_summary";
+    String OP_GET_FILE_CHECKSUM = "op_get_file_checksum";
+    String OP_GET_FILE_STATUS = "op_get_file_status";
+    String OP_GET_STATUS = "op_get_status";
+    String OP_GLOB_STATUS = "op_glob_status";
+    String OP_IS_FILE = "op_is_file";
+    String OP_IS_DIRECTORY = "op_is_directory";
+    String OP_LIST_FILES = "op_list_files";
+    String OP_LIST_LOCATED_STATUS = "op_list_located_status";
+    String OP_LIST_STATUS = "op_list_status";
+    String OP_MKDIRS = "op_mkdirs";
+    String OP_MODIFY_ACL_ENTRIES = "op_modify_acl_entries";
+    String OP_OPEN = "op_open";
+    String OP_REMOVE_ACL = "op_remove_acl";
+    String OP_REMOVE_ACL_ENTRIES = "op_remove_acl_entries";
+    String OP_REMOVE_DEFAULT_ACL = "op_remove_default_acl";
+    String OP_RENAME = "op_rename";
+    String OP_SET_ACL = "op_set_acl";
+    String OP_SET_OWNER = "op_set_owner";
+    String OP_SET_PERMISSION = "op_set_permission";
+    String OP_SET_TIMES = "op_set_times";
+    String OP_TRUNCATE = "op_truncate";
+  }
+
   /**
    * A 64-bit storage statistic.
    */
@@ -67,6 +112,14 @@ public abstract class StorageStatistics {
     return name;
   }
 
+  /**
+   * @return the associated file system scheme if this is scheme specific,
+   * else return null.
+   */
+  public String getScheme() {
+    return null;
+  }
+
   /**
    * Get an iterator over all the currently tracked long statistics.
    *

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.hdfs;
 
 import org.apache.hadoop.fs.StorageStatistics;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 
 import java.util.EnumMap;
 import java.util.Iterator;
@@ -37,55 +38,55 @@ import java.util.concurrent.atomic.AtomicLong;
  */
 public class DFSOpsCountStatistics extends StorageStatistics {
 
-  /** This is for counting file system operations. */
+  /** This is for counting distributed file system operations. */
   public enum OpType {
-    ALLOW_SNAPSHOT("allowSnapshot"),
-    APPEND("append"),
-    CONCAT("concat"),
-    COPY_FROM_LOCAL_FILE("copyFromLocalFile"),
-    CREATE("create"),
-    CREATE_NON_RECURSIVE("createNonRecursive"),
-    CREATE_SNAPSHOT("createSnapshot"),
-    CREATE_SYM_LINK("createSymlink"),
-    DELETE("delete"),
-    DELETE_SNAPSHOT("deleteSnapshot"),
-    DISALLOW_SNAPSHOT("disallowSnapshot"),
-    EXISTS("exists"),
-    GET_BYTES_WITH_FUTURE_GS("getBytesWithFutureGenerationStamps"),
-    GET_CONTENT_SUMMARY("getContentSummary"),
-    GET_FILE_BLOCK_LOCATIONS("getFileBlockLocations"),
-    GET_FILE_CHECKSUM("getFileChecksum"),
-    GET_FILE_LINK_STATUS("getFileLinkStatus"),
-    GET_FILE_STATUS("getFileStatus"),
-    GET_LINK_TARGET("getLinkTarget"),
-    GET_QUOTA_USAGE("getQuotaUsage"),
-    GET_STATUS("getStatus"),
-    GET_STORAGE_POLICIES("getStoragePolicies"),
-    GET_STORAGE_POLICY("getStoragePolicy"),
-    GET_XATTR("getXAttr"),
-    LIST_LOCATED_STATUS("listLocatedStatus"),
-    LIST_STATUS("listStatus"),
-    MKDIRS("mkdirs"),
-    MODIFY_ACL_ENTRIES("modifyAclEntries"),
-    OPEN("open"),
-    PRIMITIVE_CREATE("primitiveCreate"),
-    PRIMITIVE_MKDIR("primitiveMkdir"),
-    REMOVE_ACL("removeAcl"),
-    REMOVE_ACL_ENTRIES("removeAclEntries"),
-    REMOVE_DEFAULT_ACL("removeDefaultAcl"),
-    REMOVE_XATTR("removeXAttr"),
-    RENAME("rename"),
-    RENAME_SNAPSHOT("renameSnapshot"),
-    RESOLVE_LINK("resolveLink"),
-    SET_ACL("setAcl"),
-    SET_OWNER("setOwner"),
-    SET_PERMISSION("setPermission"),
-    SET_REPLICATION("setReplication"),
-    SET_STORAGE_POLICY("setStoragePolicy"),
-    SET_TIMES("setTimes"),
-    SET_XATTR("setXAttr"),
-    TRUNCATE("truncate"),
-    UNSET_STORAGE_POLICY("unsetStoragePolicy");
+    ALLOW_SNAPSHOT("op_allow_snapshot"),
+    APPEND(CommonStatisticNames.OP_APPEND),
+    CONCAT("op_concat"),
+    COPY_FROM_LOCAL_FILE(CommonStatisticNames.OP_COPY_FROM_LOCAL_FILE),
+    CREATE(CommonStatisticNames.OP_CREATE),
+    CREATE_NON_RECURSIVE(CommonStatisticNames.OP_CREATE_NON_RECURSIVE),
+    CREATE_SNAPSHOT("op_create_snapshot"),
+    CREATE_SYM_LINK("op_create_symlink"),
+    DELETE(CommonStatisticNames.OP_DELETE),
+    DELETE_SNAPSHOT("op_delete_snapshot"),
+    DISALLOW_SNAPSHOT("op_disallow_snapshot"),
+    EXISTS(CommonStatisticNames.OP_EXISTS),
+    GET_BYTES_WITH_FUTURE_GS("op_get_bytes_with_future_generation_stamps"),
+    GET_CONTENT_SUMMARY(CommonStatisticNames.OP_GET_CONTENT_SUMMARY),
+    GET_FILE_BLOCK_LOCATIONS("op_get_file_block_locations"),
+    GET_FILE_CHECKSUM(CommonStatisticNames.OP_GET_FILE_CHECKSUM),
+    GET_FILE_LINK_STATUS("op_get_file_link_status"),
+    GET_FILE_STATUS(CommonStatisticNames.OP_GET_FILE_STATUS),
+    GET_LINK_TARGET("op_get_link_target"),
+    GET_QUOTA_USAGE("op_get_quota_usage"),
+    GET_STATUS(CommonStatisticNames.OP_GET_STATUS),
+    GET_STORAGE_POLICIES("op_get_storage_policies"),
+    GET_STORAGE_POLICY("op_get_storage_policy"),
+    GET_XATTR("op_get_xattr"),
+    LIST_LOCATED_STATUS(CommonStatisticNames.OP_LIST_LOCATED_STATUS),
+    LIST_STATUS(CommonStatisticNames.OP_LIST_STATUS),
+    MKDIRS(CommonStatisticNames.OP_MKDIRS),
+    MODIFY_ACL_ENTRIES(CommonStatisticNames.OP_MODIFY_ACL_ENTRIES),
+    OPEN(CommonStatisticNames.OP_OPEN),
+    PRIMITIVE_CREATE("op_primitive_create"),
+    PRIMITIVE_MKDIR("op_primitive_mkdir"),
+    REMOVE_ACL(CommonStatisticNames.OP_REMOVE_ACL),
+    REMOVE_ACL_ENTRIES(CommonStatisticNames.OP_REMOVE_ACL_ENTRIES),
+    REMOVE_DEFAULT_ACL(CommonStatisticNames.OP_REMOVE_DEFAULT_ACL),
+    REMOVE_XATTR("op_remove_xattr"),
+    RENAME(CommonStatisticNames.OP_RENAME),
+    RENAME_SNAPSHOT("op_rename_snapshot"),
+    RESOLVE_LINK("op_resolve_link"),
+    SET_ACL(CommonStatisticNames.OP_SET_ACL),
+    SET_OWNER(CommonStatisticNames.OP_SET_OWNER),
+    SET_PERMISSION(CommonStatisticNames.OP_SET_PERMISSION),
+    SET_REPLICATION("op_set_replication"),
+    SET_STORAGE_POLICY("op_set_storagePolicy"),
+    SET_TIMES(CommonStatisticNames.OP_SET_TIMES),
+    SET_XATTR("op_set_xattr"),
+    TRUNCATE(CommonStatisticNames.OP_TRUNCATE),
+    UNSET_STORAGE_POLICY("op_unset_storage_policy");
 
     private final String symbol;
 
@@ -149,6 +150,11 @@ public class DFSOpsCountStatistics extends StorageStatistics {
     }
   }
 
+  @Override
+  public String getScheme() {
+    return HdfsConstants.HDFS_URI_SCHEME;
+  }
+
   @Override
   public Iterator<LongStatistic> getLongStatistics() {
     return new LongIterator();

hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestDFSOpsCountStatistics.java

@@ -31,8 +31,10 @@ import org.junit.rules.ExpectedException;
 import org.junit.rules.Timeout;
 
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -66,6 +68,19 @@ public class TestDFSOpsCountStatistics {
     }
   }
 
+  /**
+   * This is to test the the {@link OpType} symbols are unique.
+   */
+  @Test
+  public void testOpTypeSymbolsAreUnique() {
+    final Set<String> opTypeSymbols = new HashSet<>();
+    for (OpType opType : OpType.values()) {
+      assertFalse(opTypeSymbols.contains(opType.getSymbol()));
+      opTypeSymbols.add(opType.getSymbol());
+    }
+    assertEquals(OpType.values().length, opTypeSymbols.size());
+  }
+
   @Test
   public void testGetLongStatistics() {
     short iterations = 0; // number of the iter.hasNext()

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStorageStatistics.java

@@ -86,6 +86,11 @@ public class S3AStorageStatistics extends StorageStatistics {
     }
   }
 
+  @Override
+  public String getScheme() {
+    return "s3a";
+  }
+
   @Override
   public Iterator<LongStatistic> getLongStatistics() {
     return new LongIterator();

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java

@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.fs.s3a;
 
+import org.apache.hadoop.fs.StorageStatistics.CommonStatisticNames;
+
 /**
  * Statistic which are collected in S3A.
  * These statistics are available at a low level in {@link S3AStorageStatistics}
@@ -38,27 +40,27 @@ public enum Statistic {
   FILES_DELETED("files_deleted",
       "Total number of files deleted from the object store."),
   IGNORED_ERRORS("ignored_errors", "Errors caught and ignored"),
-  INVOCATION_COPY_FROM_LOCAL_FILE("invocations_copyfromlocalfile",
+  INVOCATION_COPY_FROM_LOCAL_FILE(CommonStatisticNames.OP_COPY_FROM_LOCAL_FILE,
       "Calls of copyFromLocalFile()"),
-  INVOCATION_EXISTS("invocations_exists",
+  INVOCATION_EXISTS(CommonStatisticNames.OP_EXISTS,
       "Calls of exists()"),
-  INVOCATION_GET_FILE_STATUS("invocations_getfilestatus",
+  INVOCATION_GET_FILE_STATUS(CommonStatisticNames.OP_GET_FILE_STATUS,
       "Calls of getFileStatus()"),
-  INVOCATION_GLOB_STATUS("invocations_globstatus",
+  INVOCATION_GLOB_STATUS(CommonStatisticNames.OP_GLOB_STATUS,
       "Calls of globStatus()"),
-  INVOCATION_IS_DIRECTORY("invocations_is_directory",
+  INVOCATION_IS_DIRECTORY(CommonStatisticNames.OP_IS_DIRECTORY,
       "Calls of isDirectory()"),
-  INVOCATION_IS_FILE("invocations_is_file",
+  INVOCATION_IS_FILE(CommonStatisticNames.OP_IS_FILE,
       "Calls of isFile()"),
-  INVOCATION_LIST_FILES("invocations_listfiles",
+  INVOCATION_LIST_FILES(CommonStatisticNames.OP_LIST_FILES,
       "Calls of listFiles()"),
-  INVOCATION_LIST_LOCATED_STATUS("invocations_listlocatedstatus",
+  INVOCATION_LIST_LOCATED_STATUS(CommonStatisticNames.OP_LIST_LOCATED_STATUS,
       "Calls of listLocatedStatus()"),
-  INVOCATION_LIST_STATUS("invocations_liststatus",
+  INVOCATION_LIST_STATUS(CommonStatisticNames.OP_LIST_STATUS,
       "Calls of listStatus()"),
-  INVOCATION_MKDIRS("invocations_mdkirs",
+  INVOCATION_MKDIRS(CommonStatisticNames.OP_MKDIRS,
       "Calls of mkdirs()"),
-  INVOCATION_RENAME("invocations_rename",
+  INVOCATION_RENAME(CommonStatisticNames.OP_RENAME,
       "Calls of rename()"),
   OBJECT_COPY_REQUESTS("object_copy_requests", "Object copy requests"),
   OBJECT_DELETE_REQUESTS("object_delete_requests", "Object delete requests"),