YARN-5242. Update DominantResourceCalculator to consider all resource types in calculations. Contributed by Varun Vasudev.

(cherry picked from commit 9e4ba6aff595b7a5b53d016c97c8334e10f1f2cb)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java

@@ -327,6 +327,8 @@ public abstract class Resource implements Comparable<Resource> {
     otherResources = other.getResources();
     long diff = thisResources.size() - otherResources.size();
     if (diff == 0) {
+      // compare memory and vcores first(in that order) to preserve
+      // existing behaviour
       if (thisResources.keySet().equals(otherResources.keySet())) {
         diff = this.getMemorySize() - other.getMemorySize();
         if (diff == 0) {
@@ -335,6 +337,11 @@ public abstract class Resource implements Comparable<Resource> {
         if (diff == 0) {
           for (Map.Entry<String, ResourceInformation> entry : thisResources
               .entrySet()) {
+            if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
+                || entry.getKey()
+                .equals(ResourceInformation.VCORES.getName())) {
+              continue;
+            }
             diff =
                 entry.getValue().compareTo(otherResources.get(entry.getKey()));
             if (diff != 0) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java

@@ -242,7 +242,7 @@ public class ResourcePBImpl extends Resource {
         builder.addResourceValueMap(e);
       }
     }
-    builder.setMemory(this.getMemory());
+    builder.setMemory(this.getMemorySize());
     builder.setVirtualCores(this.getVirtualCores());
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java

@@ -397,10 +397,25 @@ public class DominantResourceCalculator extends ResourceCalculator {
   }
 
   @Override
-  public boolean fitsIn(Resource cluster,
-      Resource smaller, Resource bigger) {
-    return smaller.getMemorySize() <= bigger.getMemorySize()
-        && smaller.getVirtualCores() <= bigger.getVirtualCores();
+  public boolean fitsIn(Resource cluster, Resource smaller, Resource bigger) {
+    for (String resource : resourceNames) {
+      try {
+        ResourceInformation sResourceInformation =
+            smaller.getResourceInformation(resource);
+        ResourceInformation bResourceInformation =
+            bigger.getResourceInformation(resource);
+        Long sResourceValue = UnitsConversionUtil
+            .convert(sResourceInformation.getUnits(),
+                bResourceInformation.getUnits(),
+                sResourceInformation.getValue());
+        if(sResourceValue > bResourceInformation.getValue()) {
+          return false;
+        }
+      } catch (YarnException ye) {
+        return false;
+      }
+    }
+    return true;
   }
 
   @Override

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java

@@ -178,7 +178,6 @@ public class ResourceUtils {
       synchronized (ResourceUtils.class) {
         if (lock == null) {
           synchronized (ResourceUtils.class) {
-            lock = new Object();
             Map<String, ResourceInformation> resources = new HashMap<>();
             if (conf == null) {
               conf = new YarnConfiguration();
@@ -187,10 +186,12 @@ public class ResourceUtils {
               addResourcesFileToConf(resourceFile, conf);
               LOG.debug("Found " + resourceFile + ", adding to configuration");
               initializeResourcesMap(conf, resources);
+              lock = new Object();
             } catch (FileNotFoundException fe) {
               LOG.info("Unable to find '" + resourceFile
                   + "'. Falling back to memory and vcores as resources", fe);
               initializeResourcesMap(conf, resources);
+              lock = new Object();
             }
           }
         }
@@ -268,12 +269,12 @@ public class ResourceUtils {
       synchronized (ResourceUtils.class) {
         if (nodeLock == null) {
           synchronized (ResourceUtils.class) {
-            nodeLock = new Object();
             Map<String, ResourceInformation> nodeResources =
                 initializeNodeResourceInformation(conf);
             addManadtoryResources(nodeResources);
             checkMandatatoryResources(nodeResources);
             readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
+            nodeLock = new Object();
           }
         }
       }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java

@@ -134,16 +134,12 @@ public class Resources {
 
     private Map<String, ResourceInformation> initResourceMap() {
       Map<String, ResourceInformation> tmp = new HashMap<>();
-      // Due to backwards compat, the max value for memory and vcores
-      // needs to be Integer.MAX_VALUE
-      int max = resourceValue > Integer.MAX_VALUE ? Integer.MAX_VALUE :
-          resourceValue.intValue();
       Map<String, ResourceInformation> types = ResourceUtils.getResourceTypes();
       if (types != null) {
         for (Map.Entry<String, ResourceInformation> entry : types.entrySet()) {
           tmp.put(entry.getKey(),
               ResourceInformation.newInstance(entry.getValue()));
-          tmp.get(entry.getKey()).setValue((long) max);
+          tmp.get(entry.getKey()).setValue(resourceValue);
         }
       }
       return tmp;