YARN-9124. Resolve contradiction in ResourceUtils: addMandatoryResources / checkMandatoryResources work differently (#1121)

(cherry picked from commit cbcada804d119b837ad99de71d7f44cb4629026e)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java

@@ -53,7 +53,11 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
   public static final ResourceInformation FPGAS =
           ResourceInformation.newInstance(FPGA_URI);
 
-  public static final Map<String, ResourceInformation> MANDATORY_RESOURCES =
+  /**
+   * Special resources that should be treated separately
+   * from arbitrary resource types.
+   */
+  public static final Map<String, ResourceInformation> SPECIAL_RESOURCES =
       ImmutableMap.of(MEMORY_URI, MEMORY_MB, VCORES_URI, VCORES, GPU_URI, GPUS, FPGA_URI, FPGAS);
 
   /**

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java

@@ -80,14 +80,28 @@ public class ResourceUtils {
   private ResourceUtils() {
   }
 
-  private static void checkMandatoryResources(
+  /**
+   * Ensures that historical resource types (like {@link
+   * ResourceInformation#MEMORY_URI}, {@link ResourceInformation#VCORES_URI})
+   * are not getting overridden in the resourceInformationMap.
+   *
+   * Also checks whether {@link ResourceInformation#SPECIAL_RESOURCES} are not
+   * configured poorly: having their proper units and types.
+   *
+   * @param resourceInformationMap Map object having keys as resources names
+   *                               and {@link ResourceInformation} objects as
+   *                               values
+   * @throws YarnRuntimeException if either of the two above
+   * conditions do not hold
+   */
+  private static void checkSpecialResources(
       Map<String, ResourceInformation> resourceInformationMap)
       throws YarnRuntimeException {
     /*
-     * Supporting 'memory', 'memory-mb', 'vcores' also as invalid resource names, in addition to
-     * 'MEMORY' for historical reasons
+     * Supporting 'memory', 'memory-mb', 'vcores' also as invalid resource
+     * names, in addition to 'MEMORY' for historical reasons
      */
-    String keys[] = { "memory", ResourceInformation.MEMORY_URI,
+    String[] keys = { "memory", ResourceInformation.MEMORY_URI,
         ResourceInformation.VCORES_URI };
     for(String key : keys) {
       if (resourceInformationMap.containsKey(key)) {
@@ -98,7 +112,7 @@ public class ResourceUtils {
     }
 
     for (Map.Entry<String, ResourceInformation> mandatoryResourceEntry :
-        ResourceInformation.MANDATORY_RESOURCES.entrySet()) {
+        ResourceInformation.SPECIAL_RESOURCES.entrySet()) {
       String key = mandatoryResourceEntry.getKey();
       ResourceInformation mandatoryRI = mandatoryResourceEntry.getValue();
 
@@ -121,6 +135,13 @@ public class ResourceUtils {
     }
   }
 
+  /**
+   * Ensures that {@link ResourceUtils#MEMORY} and {@link ResourceUtils#VCORES}
+   * resources are contained in the map received as parameter.
+   *
+   * @param res Map object having keys as resources names
+   *            and {@link ResourceInformation} objects as values
+   */
   private static void addMandatoryResources(
       Map<String, ResourceInformation> res) {
     ResourceInformation ri;
@@ -258,7 +279,7 @@ public class ResourceUtils {
       validateNameOfResourceNameAndThrowException(name);
     }
 
-    checkMandatoryResources(resourceInformationMap);
+    checkSpecialResources(resourceInformationMap);
     addMandatoryResources(resourceInformationMap);
 
     setAllocationForMandatoryResources(resourceInformationMap, conf);
@@ -487,7 +508,7 @@ public class ResourceUtils {
         if (!initializedNodeResources) {
           Map<String, ResourceInformation> nodeResources = initializeNodeResourceInformation(
               conf);
-          checkMandatoryResources(nodeResources);
+          checkSpecialResources(nodeResources);
           addMandatoryResources(nodeResources);
           setAllocationForMandatoryResources(nodeResources, conf);
           readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);