YARN-5587. [Partial backport] Add support for resource profiles. (vvasudev via asuresh)

(cherry picked from commit 6708ac330147b2d3816a31f2ee83e09c41fe0dd9)

hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml

@@ -153,6 +153,10 @@
     <Class name="org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceRetentionSet$LRUComparator" />
     <Bug pattern="SE_COMPARATOR_SHOULD_BE_SERIALIZABLE" />
   </Match>
+  <Match>
+    <Class name="org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl$ProfileCapabilityComparator" />
+    <Bug pattern="SE_COMPARATOR_SHOULD_BE_SERIALIZABLE" />
+  </Match>
   <Match>
     <Class name="org.apache.hadoop.yarn.exceptions.impl.pb.YarnRemoteExceptionPBImpl" />
     <Field name="builder" />

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java

@@ -19,7 +19,9 @@
 package org.apache.hadoop.yarn.api.records;
 
 import org.apache.commons.lang.NotImplementedException;
+import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
 import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
@@ -101,6 +103,18 @@ public abstract class Resource implements Comparable<Resource> {
     return new SimpleResource(memory, vCores);
   }
 
+  @InterfaceAudience.Private
+  @InterfaceStability.Unstable
+  public static Resource newInstance(Resource resource) {
+    Resource ret = Resource.newInstance(0, 0);
+    for (Map.Entry<String, ResourceInformation> entry : resource.getResources()
+        .entrySet()) {
+      ret.setResourceInformation(entry.getKey(),
+          ResourceInformation.newInstance(entry.getValue()));
+    }
+    return ret;
+  }
+
   /**
    * This method is DEPRECATED:
    * Use {@link Resource#getMemorySize()} instead

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java

@@ -31,6 +31,8 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
   private String units;
   private ResourceTypes resourceType;
   private Long value;
+  private Long minimumAllocation;
+  private Long maximumAllocation;
 
   private static final String MEMORY_URI = "memory-mb";
   private static final String VCORES_URI = "vcores";
@@ -117,6 +119,42 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
     this.value = rValue;
   }
 
+  /**
+   * Get the minimum allocation for the resource.
+   *
+   * @return the minimum allocation for the resource
+   */
+  public Long getMinimumAllocation() {
+    return minimumAllocation;
+  }
+
+  /**
+   * Set the minimum allocation for the resource.
+   *
+   * @param minimumAllocation the minimum allocation for the resource
+   */
+  public void setMinimumAllocation(Long minimumAllocation) {
+    this.minimumAllocation = minimumAllocation;
+  }
+
+  /**
+   * Get the maximum allocation for the resource.
+   *
+   * @return the maximum allocation for the resource
+   */
+  public Long getMaximumAllocation() {
+    return maximumAllocation;
+  }
+
+  /**
+   * Set the maximum allocation for the resource.
+   *
+   * @param maximumAllocation the maximum allocation for the resource
+   */
+  public void setMaximumAllocation(Long maximumAllocation) {
+    this.maximumAllocation = maximumAllocation;
+  }
+
   /**
    * Create a new instance of ResourceInformation from another object.
    *
@@ -129,33 +167,41 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
     ret.setResourceType(other.getResourceType());
     ret.setUnits(other.getUnits());
     ret.setValue(other.getValue());
+    ret.setMinimumAllocation(other.getMinimumAllocation());
+    ret.setMaximumAllocation(other.getMaximumAllocation());
     return ret;
   }
 
   public static ResourceInformation newInstance(String name, String units,
-      Long value, ResourceTypes type) {
+      Long value, ResourceTypes type, Long minimumAllocation,
+      Long maximumAllocation) {
     ResourceInformation ret = new ResourceInformation();
     ret.setName(name);
     ret.setResourceType(type);
     ret.setUnits(units);
     ret.setValue(value);
+    ret.setMinimumAllocation(minimumAllocation);
+    ret.setMaximumAllocation(maximumAllocation);
     return ret;
   }
 
   public static ResourceInformation newInstance(String name, String units,
       Long value) {
     return ResourceInformation
-        .newInstance(name, units, value, ResourceTypes.COUNTABLE);
+        .newInstance(name, units, value, ResourceTypes.COUNTABLE, 0L,
+            Long.MAX_VALUE);
   }
 
   public static ResourceInformation newInstance(String name, String units) {
     return ResourceInformation
-        .newInstance(name, units, 0L, ResourceTypes.COUNTABLE);
+        .newInstance(name, units, 0L, ResourceTypes.COUNTABLE, 0L,
+            Long.MAX_VALUE);
   }
 
   public static ResourceInformation newInstance(String name, Long value) {
     return ResourceInformation
-        .newInstance(name, "", value, ResourceTypes.COUNTABLE);
+        .newInstance(name, "", value, ResourceTypes.COUNTABLE, 0L,
+            Long.MAX_VALUE);
   }
 
   public static ResourceInformation newInstance(String name) {
@@ -165,7 +211,8 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
   @Override
   public String toString() {
     return "name: " + this.name + ", units: " + this.units + ", type: "
-        + resourceType + ", value: " + value;
+        + resourceType + ", value: " + value + ", minimum allocation: "
+        + minimumAllocation + ", maximum allocation: " + maximumAllocation;
   }
 
   public String getShorthandRepresentation() {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java

@@ -149,12 +149,7 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
   }
 
   static boolean canFit(Resource arg0, Resource arg1) {
-    long mem0 = arg0.getMemorySize();
-    long mem1 = arg1.getMemorySize();
-    long cpu0 = arg0.getVirtualCores();
-    long cpu1 = arg1.getVirtualCores();
-    
-    return (mem0 <= mem1 && cpu0 <= cpu1);
+    return Resources.fitsIn(arg0, arg1);
   }
 
   private final Map<Long, RemoteRequestsTable<T>> remoteRequests =

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java

@@ -144,8 +144,8 @@ public class ResourcePBImpl extends Resource {
               ResourceTypes.COUNTABLE;
       String units = entry.hasUnits() ? entry.getUnits() : "";
       Long value = entry.hasValue() ? entry.getValue() : 0L;
-      ResourceInformation ri =
-          ResourceInformation.newInstance(entry.getKey(), units, value, type);
+      ResourceInformation ri = ResourceInformation
+          .newInstance(entry.getKey(), units, value, type, 0L, Long.MAX_VALUE);
       if (resources.containsKey(ri.getName())) {
         resources.get(ri.getName()).setResourceType(ri.getResourceType());
         resources.get(ri.getName()).setUnits(ri.getUnits());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java

@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.conf.ConfigurationProvider;
 import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
@@ -51,6 +52,8 @@ public class ResourceUtils {
 
   public static final String UNITS = ".units";
   public static final String TYPE = ".type";
+  public static final String MINIMUM_ALLOCATION = ".minimum-allocation";
+  public static final String MAXIMUM_ALLOCATION = ".maximum-allocation";
 
   private static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
   private static final String VCORES = ResourceInformation.VCORES.getName();
@@ -122,6 +125,86 @@ public class ResourceUtils {
     }
   }
 
+  private static void setMinimumAllocationForMandatoryResources(
+      Map<String, ResourceInformation> res, Configuration conf) {
+    String[][] resourceTypesKeys =
+        {
+          { ResourceInformation.MEMORY_MB.getName(),
+            YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+            String.valueOf(
+              YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB),
+            ResourceInformation.MEMORY_MB.getName()
+          },
+          { ResourceInformation.VCORES.getName(),
+            YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+            String.valueOf(
+              YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES),
+            ResourceInformation.VCORES.getName()
+          }
+        };
+    for (String[] arr : resourceTypesKeys) {
+      String resourceTypesKey =
+          YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MINIMUM_ALLOCATION;
+      long minimumResourceTypes = conf.getLong(resourceTypesKey, -1);
+      long minimumConf = conf.getLong(arr[1], -1);
+      long minimum;
+      if (minimumResourceTypes != -1) {
+        minimum = minimumResourceTypes;
+        if (minimumConf != -1) {
+          LOG.warn("Using minimum allocation for memory specified in "
+              + "resource-types config file with key "
+              + minimumResourceTypes + ", ignoring minimum specified using "
+              + arr[1]);
+        }
+      } else {
+        minimum = conf.getLong(arr[1], Long.parseLong(arr[2]));
+      }
+      ResourceInformation ri = res.get(arr[3]);
+      ri.setMinimumAllocation(minimum);
+    }
+  }
+
+  private static void setMaximumAllocationForMandatoryResources(
+      Map<String, ResourceInformation> res, Configuration conf) {
+    String[][] resourceTypesKeys =
+        {
+          {
+            ResourceInformation.MEMORY_MB.getName(),
+            YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+            String.valueOf(
+              YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
+            ResourceInformation.MEMORY_MB.getName()
+          },
+          {
+            ResourceInformation.VCORES.getName(),
+            YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+            String.valueOf(
+              YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES),
+             ResourceInformation.VCORES.getName()
+          }
+        };
+    for (String[] arr : resourceTypesKeys) {
+      String resourceTypesKey =
+          YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MAXIMUM_ALLOCATION;
+      long maximumResourceTypes = conf.getLong(resourceTypesKey, -1);
+      long maximumConf = conf.getLong(arr[1], -1);
+      long maximum;
+      if (maximumResourceTypes != -1) {
+        maximum = maximumResourceTypes;
+        if (maximumConf != -1) {
+          LOG.warn("Using maximum allocation for memory specified in "
+              + "resource-types config file with key "
+              + maximumResourceTypes + ", ignoring maximum specified using "
+              + arr[1]);
+        }
+      } else {
+        maximum = conf.getLong(arr[1], Long.parseLong(arr[2]));
+      }
+      ResourceInformation ri = res.get(arr[3]);
+      ri.setMaximumAllocation(maximum);
+    }
+  }
+
   @VisibleForTesting
   static void initializeResourcesMap(Configuration conf,
       Map<String, ResourceInformation> resourceInformationMap) {
@@ -135,6 +218,12 @@ public class ResourceUtils {
         String resourceTypeName = conf.get(
             YarnConfiguration.RESOURCE_TYPES + "." + resourceName + TYPE,
             ResourceTypes.COUNTABLE.toString());
+        Long minimumAllocation = conf.getLong(
+            YarnConfiguration.RESOURCE_TYPES + "." + resourceName
+                + MINIMUM_ALLOCATION, 0L);
+        Long maximumAllocation = conf.getLong(
+            YarnConfiguration.RESOURCE_TYPES + "." + resourceName
+                + MAXIMUM_ALLOCATION, Long.MAX_VALUE);
         if (resourceName == null || resourceName.isEmpty()
             || resourceUnits == null || resourceTypeName == null) {
           throw new YarnRuntimeException(
@@ -154,11 +243,14 @@ public class ResourceUtils {
               "Error in config, key '" + resourceName + "' specified twice");
         }
         resourceInformationMap.put(resourceName, ResourceInformation
-            .newInstance(resourceName, resourceUnits, 0L, resourceType));
+            .newInstance(resourceName, resourceUnits, 0L, resourceType,
+                minimumAllocation, maximumAllocation));
       }
     }
     checkMandatatoryResources(resourceInformationMap);
     addManadtoryResources(resourceInformationMap);
+    setMinimumAllocationForMandatoryResources(resourceInformationMap, conf);
+    setMaximumAllocationForMandatoryResources(resourceInformationMap, conf);
     readOnlyResources = Collections.unmodifiableMap(resourceInformationMap);
   }
 
@@ -172,6 +264,12 @@ public class ResourceUtils {
         YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
   }
 
+  private static Map<String, ResourceInformation> getResourceTypes(
+      Configuration conf) {
+    return getResourceTypes(conf,
+        YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+  }
+
   private static Map<String, ResourceInformation> getResourceTypes(
       Configuration conf, String resourceFile) {
     if (lock == null) {
@@ -205,6 +303,12 @@ public class ResourceUtils {
 
     ConfigurationProvider provider =
         ConfigurationProviderFactory.getConfigurationProvider(conf);
+    try {
+      provider.init(conf);
+    } catch (Exception e) {
+      throw new IOException(e);
+    }
+
     InputStream ris = provider.getConfigurationInputStream(conf, resourceFile);
     if (ris == null) {
       if (conf.getResource(resourceFile) == null) {
@@ -241,6 +345,12 @@ public class ResourceUtils {
     lock = null;
   }
 
+  @VisibleForTesting
+  public static void resetResourceTypes(Configuration conf) {
+    lock = null;
+    getResourceTypes(conf);
+  }
+
   public static String getUnits(String resourceValue) {
     String units;
     for (int i = 0; i < resourceValue.length(); i++) {
@@ -326,4 +436,53 @@ public class ResourceUtils {
     nodeLock = null;
   }
 
+  public static Resource getResourceTypesMinimumAllocation() {
+    Map<String, ResourceInformation> resourceTypes = getResourceTypes();
+    Resource ret = Resource.newInstance(0, 0);
+    for (Map.Entry<String, ResourceInformation> entry : resourceTypes
+        .entrySet()) {
+      String name = entry.getKey();
+      if (name.equals(ResourceInformation.MEMORY_MB.getName())) {
+        ret.setMemorySize(entry.getValue().getMinimumAllocation());
+        continue;
+      }
+      if (name.equals(ResourceInformation.VCORES.getName())) {
+        Long tmp = entry.getValue().getMinimumAllocation();
+        if (tmp > Integer.MAX_VALUE) {
+          tmp = (long) Integer.MAX_VALUE;
+        }
+        ret.setVirtualCores(tmp.intValue());
+        continue;
+      }
+      ret.setResourceValue(name, entry.getValue().getMinimumAllocation());
+    }
+    return ret;
+  }
+
+  /**
+   * Get a Resource object with for the maximum allocation possible.
+   * @return a Resource object with the maximum allocation for the scheduler
+   */
+  public static Resource getResourceTypesMaximumAllocation() {
+    Map<String, ResourceInformation> resourceTypes = getResourceTypes();
+    Resource ret = Resource.newInstance(0, 0);
+    for (Map.Entry<String, ResourceInformation> entry : resourceTypes
+        .entrySet()) {
+      String name = entry.getKey();
+      if (name.equals(ResourceInformation.MEMORY_MB.getName())) {
+        ret.setMemorySize(entry.getValue().getMaximumAllocation());
+        continue;
+      }
+      if (name.equals(ResourceInformation.VCORES.getName())) {
+        Long tmp = entry.getValue().getMaximumAllocation();
+        if (tmp > Integer.MAX_VALUE) {
+          tmp = (long) Integer.MAX_VALUE;
+        }
+        ret.setVirtualCores(tmp.intValue());
+        continue;
+      }
+      ret.setResourceValue(name, entry.getValue().getMaximumAllocation());
+    }
+    return ret;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java

@@ -75,12 +75,12 @@ public class Resources {
     }
 
     @Override
+    @SuppressWarnings("deprecation")
     public void setMemory(int memory) {
       throw new RuntimeException(name + " cannot be modified!");
     }
 
     @Override
-    @SuppressWarnings("deprecation")
     public void setMemorySize(long memory) {
       throw new RuntimeException(name + " cannot be modified!");
     }
@@ -193,13 +193,7 @@ public class Resources {
   }
 
   public static Resource clone(Resource res) {
-    Resource ret = Resource.newInstance(0, 0);
-    for (Map.Entry<String, ResourceInformation> entry : res.getResources()
-        .entrySet()) {
-      ret.setResourceInformation(entry.getKey(),
-          ResourceInformation.newInstance(entry.getValue()));
-    }
-    return ret;
+    return Resource.newInstance(res);
   }
 
   public static Resource addTo(Resource lhs, Resource rhs) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java

@@ -96,6 +96,7 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.server.utils.Lock;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -1321,8 +1322,51 @@ public abstract class AbstractYarnScheduler
    * @param container Container.
    */
   public void asyncContainerRelease(RMContainer container) {
-    this.rmContext.getDispatcher().getEventHandler()
-        .handle(new ReleaseContainerEvent(container));
+    this.rmContext.getDispatcher().getEventHandler().handle(
+        new ReleaseContainerEvent(container));
+  }
+
+  /*
+   * Get a Resource object with for the minimum allocation possible. If resource
+   * profiles are enabled then the 'minimum' resource profile will be used. If
+   * they are not enabled, use the minimums specified in the config files.
+   *
+   * @return a Resource object with the minimum allocation for the scheduler
+   */
+  public Resource getMinimumAllocation() {
+    boolean profilesEnabled = getConfig()
+        .getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
+            YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
+    Resource ret;
+    if (!profilesEnabled) {
+      ret = ResourceUtils.getResourceTypesMinimumAllocation();
+    } else {
+      ret = rmContext.getResourceProfilesManager().getMinimumProfile();
+    }
+    LOG.info("Minimum allocation = " + ret);
+    return ret;
+  }
+
+  /**
+   * Get a Resource object with for the maximum allocation possible. If resource
+   * profiles are enabled then the 'maximum' resource profile will be used. If
+   * they are not enabled, use the maximums specified in the config files.
+   *
+   * @return a Resource object with the maximum allocation for the scheduler
+   */
+
+  public Resource getMaximumAllocation() {
+    boolean profilesEnabled = getConfig()
+        .getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
+            YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
+    Resource ret;
+    if (!profilesEnabled) {
+      ret = ResourceUtils.getResourceTypesMaximumAllocation();
+    } else {
+      ret = rmContext.getResourceProfilesManager().getMaximumProfile();
+    }
+    LOG.info("Maximum allocation = " + ret);
+    return ret;
   }
 
   @Override

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java

@@ -221,8 +221,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
 
       return Resources.createResource(
           Math.min(configuredMaxAllocation.getMemorySize(), maxNodeMemory),
-          Math.min(configuredMaxAllocation.getVirtualCores(), maxNodeVCores)
-      );
+          Math.min(configuredMaxAllocation.getVirtualCores(), maxNodeVCores));
     } finally {
       readLock.unlock();
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java

@@ -315,8 +315,8 @@ public class CapacityScheduler extends
       this.csConfProvider.init(configuration);
       this.conf = this.csConfProvider.loadConfiguration(configuration);
       validateConf(this.conf);
-      this.minimumAllocation = this.conf.getMinimumAllocation();
-      initMaximumResourceCapability(this.conf.getMaximumAllocation());
+      this.minimumAllocation = super.getMinimumAllocation();
+      initMaximumResourceCapability(super.getMaximumAllocation());
       this.calculator = this.conf.getResourceCalculator();
       this.usePortForNodeName = this.conf.getUsePortForNodeName();
       this.applications = new ConcurrentHashMap<>();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java

@@ -1258,8 +1258,8 @@ public class FairScheduler extends
       this.conf = new FairSchedulerConfiguration(conf);
       validateConf(this.conf);
       authorizer = YarnAuthorizationProvider.getInstance(conf);
-      minimumAllocation = this.conf.getMinimumAllocation();
-      initMaximumResourceCapability(this.conf.getMaximumAllocation());
+      minimumAllocation = super.getMinimumAllocation();
+      initMaximumResourceCapability(super.getMaximumAllocation());
       incrAllocation = this.conf.getIncrementAllocation();
       updateReservationThreshold();
       continuousSchedulingEnabled = this.conf.isContinuousSchedulingEnabled();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java

@@ -241,17 +241,8 @@ public class FifoScheduler extends
     //Use ConcurrentSkipListMap because applications need to be ordered
     this.applications =
         new ConcurrentSkipListMap<>();
-    this.minimumAllocation =
-        Resources.createResource(conf.getInt(
-            YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
-    initMaximumResourceCapability(
-        Resources.createResource(conf.getInt(
-            YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
-          conf.getInt(
-            YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)));
+    this.minimumAllocation = super.getMinimumAllocation();
+    initMaximumResourceCapability(super.getMaximumAllocation());
     this.usePortForNodeName = conf.getBoolean(
         YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME,
         YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java

@@ -104,6 +104,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManag
 import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
 import org.apache.hadoop.yarn.util.Records;
 import org.apache.hadoop.yarn.util.YarnVersionInfo;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.log4j.Level;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
@@ -150,6 +151,7 @@ public class MockRM extends ResourceManager {
   public MockRM(Configuration conf, RMStateStore store,
       boolean useNullRMNodeLabelsManager, boolean useRealElector) {
     super();
+    ResourceUtils.resetResourceTypes(conf);
     this.useNullRMNodeLabelsManager = useNullRMNodeLabelsManager;
     this.useRealElector = useRealElector;
     init(conf instanceof YarnConfiguration ? conf : new YarnConfiguration(conf));

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java

@@ -247,6 +247,7 @@ public class TestAppManager{
     ResourceScheduler scheduler = mockResourceScheduler();
     ((RMContextImpl)rmContext).setScheduler(scheduler);
     Configuration conf = new Configuration();
+    ((RMContextImpl) rmContext).setYarnConfiguration(conf);
     ApplicationMasterService masterService =
         new ApplicationMasterService(rmContext, scheduler);
     appMonitor = new TestRMAppManager(rmContext,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java

@@ -113,6 +113,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.Dom
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.util.ControlledClock;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.After;
 import org.junit.Assert;
@@ -212,6 +213,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
     conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
     conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 
       128);
+    ResourceUtils.resetResourceTypes(conf);
     scheduler.init(conf);
     scheduler.start();
     scheduler.reinitialize(conf, resourceManager.getRMContext());
@@ -240,6 +242,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
       FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512);
     conf.setInt(
       FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 2);
+    ResourceUtils.resetResourceTypes(conf);
     scheduler.init(conf);
     scheduler.reinitialize(conf, null);
     Assert.assertEquals(256, scheduler.getMinimumResourceCapability().getMemorySize());
@@ -257,6 +260,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
       FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512);
     conf.setInt(
       FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 2);
+    ResourceUtils.resetResourceTypes(conf);
     scheduler.init(conf);
     scheduler.reinitialize(conf, null);
     Assert.assertEquals(0, scheduler.getMinimumResourceCapability().getMemorySize());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java

@@ -93,6 +93,7 @@ import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
 import org.apache.hadoop.yarn.server.timeline.TimelineStore;
 import org.apache.hadoop.yarn.server.timeline.recovery.MemoryTimelineStateStore;
 import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
@@ -251,6 +252,7 @@ public class MiniYARNCluster extends CompositeService {
         YarnConfiguration.DEFAULT_YARN_MINICLUSTER_USE_RPC);
     failoverTimeout = conf.getInt(YarnConfiguration.RM_ZK_TIMEOUT_MS,
         YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS);
+    ResourceUtils.resetResourceTypes(conf);
 
     if (useRpc && !useFixedPorts) {
       throw new YarnRuntimeException("Invalid configuration!" +