YARN-7030. [YARN-3926] Performance optimizations in Resource and ResourceUtils class. Contributed by Wangda Tan.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java

@@ -65,7 +65,7 @@ public abstract class Resource implements Comparable<Resource> {
   @Public
   @Stable
   public static Resource newInstance(int memory, int vCores) {
-    if (ResourceUtils.getResourceTypesArray().length > 2) {
+    if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
       Resource ret = Records.newRecord(Resource.class);
       ret.setMemorySize(memory);
       ret.setVirtualCores(vCores);
@@ -77,7 +77,7 @@ public abstract class Resource implements Comparable<Resource> {
   @Public
   @Stable
   public static Resource newInstance(long memory, int vCores) {
-    if (ResourceUtils.getResourceTypesArray().length > 2) {
+    if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
       Resource ret = Records.newRecord(Resource.class);
       ret.setMemorySize(memory);
       ret.setVirtualCores(vCores);
@@ -91,7 +91,7 @@ public abstract class Resource implements Comparable<Resource> {
   public static Resource newInstance(Resource resource) {
     Resource ret = Resource.newInstance(resource.getMemorySize(),
         resource.getVirtualCores());
-    if (ResourceUtils.getResourceTypesArray().length > 2) {
+    if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
       Resource.copy(resource, ret);
     }
     return ret;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java

@@ -63,6 +63,11 @@ public class BaseResource extends Resource {
   protected ResourceInformation[] resources = null;
   protected ResourceInformation[] readOnlyResources = null;
 
+  // Number of mandatory resources, this is added to avoid invoke
+  // MandatoryResources.values().length, since values() internally will
+  // copy array, etc.
+  private static final int NUM_MANDATORY_RESOURCES = 2;
+
   protected enum MandatoryResources {
     MEMORY(0), VCORES(1);
 
@@ -86,9 +91,8 @@ public class BaseResource extends Resource {
         ResourceInformation.MEMORY_MB.getUnits(), memory);
     this.vcoresResInfo = ResourceInformation.newInstance(VCORES, "", vcores);
 
-    resources = new ResourceInformation[MandatoryResources.values().length];
-    readOnlyResources = new ResourceInformation[MandatoryResources
-        .values().length];
+    resources = new ResourceInformation[NUM_MANDATORY_RESOURCES];
+    readOnlyResources = new ResourceInformation[NUM_MANDATORY_RESOURCES];
     resources[MandatoryResources.MEMORY.id] = memoryResInfo;
     resources[MandatoryResources.VCORES.id] = vcoresResInfo;
     readOnlyResources = Arrays.copyOf(resources, resources.length);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java

@@ -74,6 +74,7 @@ public class ResourceUtils {
   private static volatile ResourceInformation[] resourceTypesArray;
   private static volatile boolean initializedNodeResources = false;
   private static volatile Map<String, ResourceInformation> readOnlyNodeResources;
+  private static volatile int numKnownResourceTypes = -1;
 
   static final Log LOG = LogFactory.getLog(ResourceUtils.class);
 
@@ -308,23 +309,33 @@ public class ResourceUtils {
    * @return resourceNamesArray
    */
   public static String[] getResourceNamesArray() {
-    getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+    initializeResourceTypesIfNeeded(null,
+        YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
     return resourceNamesArray;
   }
 
   public static ResourceInformation[] getResourceTypesArray() {
-    getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+    initializeResourceTypesIfNeeded(null,
+        YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
     return resourceTypesArray;
   }
 
+  public static int getNumberOfKnownResourceTypes() {
+    if (numKnownResourceTypes < 0) {
+      initializeResourceTypesIfNeeded(null,
+          YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+    }
+    return numKnownResourceTypes;
+  }
+
   private static Map<String, ResourceInformation> getResourceTypes(
       Configuration conf) {
     return getResourceTypes(conf,
         YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
   }
 
-  private static Map<String, ResourceInformation> getResourceTypes(
-      Configuration conf, String resourceFile) {
+  private static void initializeResourceTypesIfNeeded(Configuration conf,
+      String resourceFile) {
     if (!initializedResources) {
       synchronized (ResourceUtils.class) {
         if (!initializedResources) {
@@ -346,6 +357,12 @@ public class ResourceUtils {
         }
       }
     }
+    numKnownResourceTypes = resourceTypes.size();
+  }
+
+  private static Map<String, ResourceInformation> getResourceTypes(
+      Configuration conf, String resourceFile) {
+    initializeResourceTypesIfNeeded(conf, resourceFile);
     return resourceTypes;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java

@@ -173,7 +173,7 @@ public class Resources {
   }
 
   public static Resource addTo(Resource lhs, Resource rhs) {
-    int maxLength = ResourceUtils.getResourceTypesArray().length;
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
     for (int i = 0; i < maxLength; i++) {
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(i);
@@ -197,7 +197,7 @@ public class Resources {
   }
 
   public static Resource subtractFrom(Resource lhs, Resource rhs) {
-    int maxLength = ResourceUtils.getResourceTypesArray().length;
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
     for (int i = 0; i < maxLength; i++) {
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(i);
@@ -243,7 +243,7 @@ public class Resources {
   }
 
   public static Resource multiplyTo(Resource lhs, double by) {
-    int maxLength = ResourceUtils.getResourceTypesArray().length;
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
     for (int i = 0; i < maxLength; i++) {
       try {
         ResourceInformation lhsValue = lhs.getResourceInformation(i);
@@ -266,7 +266,7 @@ public class Resources {
    */
   public static Resource multiplyAndAddTo(
       Resource lhs, Resource rhs, double by) {
-    int maxLength = ResourceUtils.getResourceTypesArray().length;
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
     for (int i = 0; i < maxLength; i++) {
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(i);
@@ -299,7 +299,7 @@ public class Resources {
   
   public static Resource multiplyAndRoundDown(Resource lhs, double by) {
     Resource out = clone(lhs);
-    int maxLength = ResourceUtils.getResourceTypesArray().length;
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
     for (int i = 0; i < maxLength; i++) {
       try {
         ResourceInformation lhsValue = lhs.getResourceInformation(i);
@@ -408,7 +408,7 @@ public class Resources {
   }
   
   public static boolean fitsIn(Resource smaller, Resource bigger) {
-    int maxLength = ResourceUtils.getResourceTypesArray().length;
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
     for (int i = 0; i < maxLength; i++) {
       try {
         ResourceInformation rhsValue = bigger.getResourceInformation(i);
@@ -436,7 +436,7 @@ public class Resources {
   
   public static Resource componentwiseMin(Resource lhs, Resource rhs) {
     Resource ret = createResource(0);
-    int maxLength = ResourceUtils.getResourceTypesArray().length;
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
     for (int i = 0; i < maxLength; i++) {
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(i);
@@ -460,7 +460,7 @@ public class Resources {
   
   public static Resource componentwiseMax(Resource lhs, Resource rhs) {
     Resource ret = createResource(0);
-    int maxLength = ResourceUtils.getResourceTypesArray().length;
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
     for (int i = 0; i < maxLength; i++) {
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(i);