YARN-7159. Normalize unit of resource objects in RM to avoid unit conversion in critical path. Contributed by Manikandan R.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java

@@ -202,7 +202,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
     ResourceInformation ret = new ResourceInformation();
     ret.setName(name);
     ret.setResourceType(type);
-    ret.setUnits(units);
+    ret.setUnitsWithoutValidation(units);
     ret.setValue(value);
     ret.setMinimumAllocation(minimumAllocation);
     ret.setMaximumAllocation(maximumAllocation);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java

@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.util.UnitsConversionUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -527,6 +528,12 @@ public class ResourceUtils {
       String units = getUnits(value);
       Long resourceValue =
           Long.valueOf(value.substring(0, value.length() - units.length()));
+      String destUnit = getDefaultUnit(resourceType);
+      if(!units.equals(destUnit)) {
+        resourceValue = UnitsConversionUtil.convert(
+            units, destUnit, resourceValue);
+        units = destUnit;
+      }
       nodeResources.get(resourceType).setValue(resourceValue);
       nodeResources.get(resourceType).setUnits(units);
       if (LOG.isDebugEnabled()) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java

@@ -43,7 +43,7 @@ public class TestResourceInformation {
     Assert.assertEquals("Resource units incorrect", units, ri.getUnits());
     units = "z";
     try {
-      ResourceInformation.newInstance(name, units);
+      ResourceInformation.newInstance(name, units).setUnits(units);
       Assert.fail(units + "is not a valid unit");
     } catch (IllegalArgumentException ie) {
       // do nothing

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java

@@ -173,9 +173,17 @@ public class ResourcePBImpl extends Resource {
     ri.setResourceType(entry.hasType()
         ? ProtoUtils.convertFromProtoFormat(entry.getType())
         : ResourceTypes.COUNTABLE);
-    ri.setUnits(
-        entry.hasUnits() ? entry.getUnits() : resourceInformation.getUnits());
-    ri.setValue(entry.hasValue() ? entry.getValue() : 0L);
+    String units = entry.hasUnits() ? entry.getUnits() :
+        ResourceUtils.getDefaultUnit(entry.getKey());
+    long value = entry.hasValue() ? entry.getValue() : 0L;
+    String destUnit = ResourceUtils.getDefaultUnit(entry.getKey());
+    if(!units.equals(destUnit)) {
+      ri.setValue(UnitsConversionUtil.convert(units, destUnit, value));
+      ri.setUnits(destUnit);
+    } else {
+      ri.setUnits(units);
+      ri.setValue(value);
+    }
     return ri;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java

@@ -24,7 +24,6 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.util.UnitsConversionUtil;
 
 import java.io.PrintWriter;
 import java.io.StringWriter;
@@ -298,11 +297,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
    */
   private double calculateShare(ResourceInformation clusterRes,
       ResourceInformation res) {
-      // Convert the resources' units into the cluster resource's units
-    long value = UnitsConversionUtil.convert(res.getUnits(),
-          clusterRes.getUnits(), res.getValue());
-
-    return (double) value / clusterRes.getValue();
+    return (double) res.getValue() / clusterRes.getValue();
   }
 
   /**
@@ -340,11 +335,8 @@ public class DominantResourceCalculator extends ResourceCalculator {
       ResourceInformation availableResource = available
           .getResourceInformation(i);
       ResourceInformation requiredResource = required.getResourceInformation(i);
-      long requiredResourceValue = UnitsConversionUtil.convert(
-          requiredResource.getUnits(), availableResource.getUnits(),
-          requiredResource.getValue());
-      if (requiredResourceValue != 0) {
-        long tmp = availableResource.getValue() / requiredResourceValue;
+      if (requiredResource.getValue() != 0) {
+        long tmp = availableResource.getValue() / requiredResource.getValue();
         min = min < tmp ? min : tmp;
       }
     }
@@ -387,11 +379,8 @@ public class DominantResourceCalculator extends ResourceCalculator {
     for (int i = 0; i < maxLength; i++) {
       ResourceInformation aResourceInformation = a.getResourceInformation(i);
       ResourceInformation bResourceInformation = b.getResourceInformation(i);
-      long bResourceValue = UnitsConversionUtil.convert(
-          bResourceInformation.getUnits(), aResourceInformation.getUnits(),
-          bResourceInformation.getValue());
       float tmp = (float) aResourceInformation.getValue()
-          / (float) bResourceValue;
+          / (float) bResourceInformation.getValue();
       ratio = ratio > tmp ? ratio : tmp;
     }
     return ratio;
@@ -437,23 +426,11 @@ public class DominantResourceCalculator extends ResourceCalculator {
       ResourceInformation tmp = ret.getResourceInformation(i);
 
       long rValue = rResourceInformation.getValue();
-      long minimumValue = UnitsConversionUtil.convert(
-          minimumResourceInformation.getUnits(),
-          rResourceInformation.getUnits(),
-          minimumResourceInformation.getValue());
-      long maximumValue = UnitsConversionUtil.convert(
-          maximumResourceInformation.getUnits(),
-          rResourceInformation.getUnits(),
-          maximumResourceInformation.getValue());
-      long stepFactorValue = UnitsConversionUtil.convert(
-          stepFactorResourceInformation.getUnits(),
-          rResourceInformation.getUnits(),
-          stepFactorResourceInformation.getValue());
-      long value = Math.max(rValue, minimumValue);
-      if (stepFactorValue != 0) {
-        value = roundUp(value, stepFactorValue);
+      long value = Math.max(rValue, minimumResourceInformation.getValue());
+      if (stepFactorResourceInformation.getValue() != 0) {
+        value = roundUp(value, stepFactorResourceInformation.getValue());
       }
-      tmp.setValue(Math.min(value, maximumValue));
+      tmp.setValue(Math.min(value, maximumResourceInformation.getValue()));
       ret.setResourceInformation(i, tmp);
     }
     return ret;
@@ -478,10 +455,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
           .getResourceInformation(i);
 
       long rValue = rResourceInformation.getValue();
-      long stepFactorValue = UnitsConversionUtil.convert(
-          stepFactorResourceInformation.getUnits(),
-          rResourceInformation.getUnits(),
-          stepFactorResourceInformation.getValue());
+      long stepFactorValue = stepFactorResourceInformation.getValue();
       long value = rValue;
       if (stepFactorValue != 0) {
         value = roundUp
@@ -506,10 +480,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
           .getResourceInformation(i);
 
       long rValue = rResourceInformation.getValue();
-      long stepFactorValue = UnitsConversionUtil.convert(
-          stepFactorResourceInformation.getUnits(),
-          rResourceInformation.getUnits(),
-          stepFactorResourceInformation.getValue());
+      long stepFactorValue = stepFactorResourceInformation.getValue();
       ret.setResourceValue(i, ResourceCalculator
           .roundUp((long) Math.ceil(rValue * by[i]), stepFactorValue));
     }
@@ -539,10 +510,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
       ResourceInformation tmp = ret.getResourceInformation(i);
 
       long rValue = rResourceInformation.getValue();
-      long stepFactorValue = UnitsConversionUtil.convert(
-          stepFactorResourceInformation.getUnits(),
-          rResourceInformation.getUnits(),
-          stepFactorResourceInformation.getValue());
+      long stepFactorValue = stepFactorResourceInformation.getValue();
       long value;
       if (stepFactorValue != 0) {
         value = roundUp
@@ -566,10 +534,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
           .getResourceInformation(i);
       ResourceInformation bResourceInformation = bigger
           .getResourceInformation(i);
-      long sResourceValue = UnitsConversionUtil.convert(
-          sResourceInformation.getUnits(), bResourceInformation.getUnits(),
-          sResourceInformation.getValue());
-      if (sResourceValue > bResourceInformation.getValue()) {
+      if (sResourceInformation.getValue() > bResourceInformation.getValue()) {
         return false;
       }
     }
@@ -587,11 +552,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
       ResourceInformation tmp = ret.getResourceInformation(i);
 
       long rValue = rResourceInformation.getValue();
-      long stepFactorValue = UnitsConversionUtil.convert(
-          stepFactorResourceInformation.getUnits(),
-          rResourceInformation.getUnits(),
-          stepFactorResourceInformation.getValue());
-
+      long stepFactorValue = stepFactorResourceInformation.getValue();
       long value = rValue;
       if (stepFactorValue != 0) {
         value = roundDown(rValue, stepFactorValue);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java

@@ -27,7 +27,6 @@ import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.api.records.impl.LightWeightResource;
 import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
-import org.apache.hadoop.yarn.util.UnitsConversionUtil;
 
 /**
  * Resources is a computation class which provides a set of apis to do
@@ -257,12 +256,7 @@ public class Resources {
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(i);
         ResourceInformation lhsValue = lhs.getResourceInformation(i);
-
-        long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
-            ? rhsValue.getValue()
-            : UnitsConversionUtil.convert(rhsValue.getUnits(),
-                lhsValue.getUnits(), rhsValue.getValue());
-        lhs.setResourceValue(i, lhsValue.getValue() + convertedRhs);
+        lhs.setResourceValue(i, lhsValue.getValue() + rhsValue.getValue());
       } catch (ResourceNotFoundException ye) {
         LOG.warn("Resource is missing:" + ye.getMessage());
         continue;
@@ -281,12 +275,7 @@ public class Resources {
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(i);
         ResourceInformation lhsValue = lhs.getResourceInformation(i);
-
-        long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
-            ? rhsValue.getValue()
-            : UnitsConversionUtil.convert(rhsValue.getUnits(),
-                lhsValue.getUnits(), rhsValue.getValue());
-        lhs.setResourceValue(i, lhsValue.getValue() - convertedRhs);
+        lhs.setResourceValue(i, lhsValue.getValue() - rhsValue.getValue());
       } catch (ResourceNotFoundException ye) {
         LOG.warn("Resource is missing:" + ye.getMessage());
         continue;
@@ -365,12 +354,7 @@ public class Resources {
         ResourceInformation rhsValue = rhs.getResourceInformation(i);
         ResourceInformation lhsValue = lhs.getResourceInformation(i);
 
-        long convertedRhs = (long) (((rhsValue.getUnits()
-            .equals(lhsValue.getUnits()))
-                ? rhsValue.getValue()
-                : UnitsConversionUtil.convert(rhsValue.getUnits(),
-                    lhsValue.getUnits(), rhsValue.getValue()))
-            * by);
+        long convertedRhs = (long) (rhsValue.getValue() * by);
         lhs.setResourceValue(i, lhsValue.getValue() + convertedRhs);
       } catch (ResourceNotFoundException ye) {
         LOG.warn("Resource is missing:" + ye.getMessage());
@@ -511,12 +495,7 @@ public class Resources {
       try {
         ResourceInformation rhsValue = bigger.getResourceInformation(i);
         ResourceInformation lhsValue = smaller.getResourceInformation(i);
-
-        long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
-            ? rhsValue.getValue()
-            : UnitsConversionUtil.convert(rhsValue.getUnits(),
-                lhsValue.getUnits(), rhsValue.getValue());
-        if (lhsValue.getValue() > convertedRhs) {
+        if (lhsValue.getValue() > rhsValue.getValue()) {
           return false;
         }
       } catch (ResourceNotFoundException ye) {
@@ -539,12 +518,7 @@ public class Resources {
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(i);
         ResourceInformation lhsValue = lhs.getResourceInformation(i);
-
-        long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
-            ? rhsValue.getValue()
-            : UnitsConversionUtil.convert(rhsValue.getUnits(),
-                lhsValue.getUnits(), rhsValue.getValue());
-        ResourceInformation outInfo = lhsValue.getValue() < convertedRhs
+        ResourceInformation outInfo = lhsValue.getValue() < rhsValue.getValue()
             ? lhsValue
             : rhsValue;
         ret.setResourceInformation(i, outInfo);
@@ -563,12 +537,7 @@ public class Resources {
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(i);
         ResourceInformation lhsValue = lhs.getResourceInformation(i);
-
-        long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
-            ? rhsValue.getValue()
-            : UnitsConversionUtil.convert(rhsValue.getUnits(),
-                lhsValue.getUnits(), rhsValue.getValue());
-        ResourceInformation outInfo = lhsValue.getValue() > convertedRhs
+        ResourceInformation outInfo = lhsValue.getValue() > rhsValue.getValue()
             ? lhsValue
             : rhsValue;
         ret.setResourceInformation(i, outInfo);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java

@@ -18,11 +18,19 @@
 
 package org.apache.hadoop.yarn.api;
 
+import java.io.File;
+
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.proto.YarnProtos;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.util.resource.TestResourceUtils;
+import org.junit.After;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
@@ -31,6 +39,27 @@ import static org.junit.Assert.assertEquals;
  * Test class to handle various proto related tests for resources.
  */
 public class TestResourcePBImpl {
+
+  @Before
+  public void setup() throws Exception {
+    ResourceUtils.resetResourceTypes();
+
+    String resourceTypesFile = "resource-types-4.xml";
+    Configuration conf = new YarnConfiguration();
+    TestResourceUtils.setupResourceTypes(conf, resourceTypesFile);
+  }
+
+  @After
+  public void teardown() {
+    Configuration conf = new YarnConfiguration();
+    File source = new File(
+        conf.getClassLoader().getResource("resource-types-4.xml").getFile());
+    File dest = new File(source.getParent(), "resource-types.xml");
+    if (dest.exists()) {
+      dest.delete();
+    }
+  }
+
   @Test
   public void testEmptyResourcePBInit() throws Exception {
     Resource res = new ResourcePBImpl();
@@ -85,4 +114,65 @@ public class TestResourcePBImpl {
     assertEquals("Cast to Integer.MAX_VALUE if the long is greater than "
         + "Integer.MAX_VALUE", Integer.MAX_VALUE, res.getVirtualCores());
   }
+
+  @Test
+  public void testResourcePBWithExtraResources() throws Exception {
+
+    //Resource 'resource1' has been passed as 4T
+    //4T should be converted to 4000G
+    YarnProtos.ResourceInformationProto riProto =
+        YarnProtos.ResourceInformationProto.newBuilder().setType(
+            YarnProtos.ResourceTypeInfoProto.newBuilder().
+            setName("resource1").setType(
+                YarnProtos.ResourceTypesProto.COUNTABLE).getType()).
+        setValue(4).setUnits("T").setKey("resource1").build();
+
+    YarnProtos.ResourceProto proto =
+        YarnProtos.ResourceProto.newBuilder().setMemory(1024).
+        setVirtualCores(3).addResourceValueMap(riProto).build();
+    Resource res = new ResourcePBImpl(proto);
+
+    Assert.assertEquals(4000,
+        res.getResourceInformation("resource1").getValue());
+    Assert.assertEquals("G",
+        res.getResourceInformation("resource1").getUnits());
+
+    //Resource 'resource2' has been passed as 4M
+    //4M should be converted to 4000000000m
+    YarnProtos.ResourceInformationProto riProto1 =
+        YarnProtos.ResourceInformationProto.newBuilder().setType(
+            YarnProtos.ResourceTypeInfoProto.newBuilder().
+            setName("resource2").setType(
+                YarnProtos.ResourceTypesProto.COUNTABLE).getType()).
+        setValue(4).setUnits("M").setKey("resource2").build();
+
+    YarnProtos.ResourceProto proto1 =
+        YarnProtos.ResourceProto.newBuilder().setMemory(1024).
+        setVirtualCores(3).addResourceValueMap(riProto1).build();
+    Resource res1 = new ResourcePBImpl(proto1);
+
+    Assert.assertEquals(4000000000L,
+        res1.getResourceInformation("resource2").getValue());
+    Assert.assertEquals("m",
+        res1.getResourceInformation("resource2").getUnits());
+
+    //Resource 'resource1' has been passed as 3M
+    //3M should be converted to 0G
+    YarnProtos.ResourceInformationProto riProto2 =
+        YarnProtos.ResourceInformationProto.newBuilder().setType(
+            YarnProtos.ResourceTypeInfoProto.newBuilder().
+            setName("resource1").setType(
+                YarnProtos.ResourceTypesProto.COUNTABLE).getType()).
+        setValue(3).setUnits("M").setKey("resource1").build();
+
+    YarnProtos.ResourceProto proto2 =
+        YarnProtos.ResourceProto.newBuilder().setMemory(1024).
+        setVirtualCores(3).addResourceValueMap(riProto2).build();
+    Resource res2 = new ResourcePBImpl(proto2);
+
+    Assert.assertEquals(0,
+        res2.getResourceInformation("resource1").getValue());
+    Assert.assertEquals("G",
+        res2.getResourceInformation("resource1").getUnits());
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java

@@ -377,6 +377,46 @@ public class TestResourceUtils {
     }
   }
 
+  @Test
+  public void testGetResourceInformationWithDiffUnits() throws Exception {
+
+    Configuration conf = new YarnConfiguration();
+    Map<String, Resource> testRun = new HashMap<>();
+    setupResourceTypes(conf, "resource-types-4.xml");
+    Resource test3Resources = Resource.newInstance(0, 0);
+
+    //Resource 'resource1' has been passed as 5T
+    //5T should be converted to 5000G
+    test3Resources.setResourceInformation("resource1",
+        ResourceInformation.newInstance("resource1", "T", 5L));
+
+    //Resource 'resource2' has been passed as 2M
+    //2M should be converted to 2000000000m
+    test3Resources.setResourceInformation("resource2",
+        ResourceInformation.newInstance("resource2", "M", 2L));
+    test3Resources.setResourceInformation("yarn.io/gpu",
+        ResourceInformation.newInstance("yarn.io/gpu", "", 1));
+    testRun.put("node-resources-3.xml", test3Resources);
+
+    for (Map.Entry<String, Resource> entry : testRun.entrySet()) {
+      String resourceFile = entry.getKey();
+      ResourceUtils.resetNodeResources();
+      File dest;
+      File source = new File(
+          conf.getClassLoader().getResource(resourceFile).getFile());
+      dest = new File(source.getParent(), "node-resources.xml");
+      FileUtils.copyFile(source, dest);
+      Map<String, ResourceInformation> actual = ResourceUtils
+          .getNodeResourceInformation(conf);
+      Assert.assertEquals(actual.size(),
+          entry.getValue().getResources().length);
+      for (ResourceInformation resInfo : entry.getValue().getResources()) {
+        Assert.assertEquals(resInfo, actual.get(resInfo.getName()));
+      }
+      dest.delete();
+    }
+  }
+
   public static String setupResourceTypes(Configuration conf, String filename)
       throws Exception {
     File source = new File(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-3.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+
+ <property>
+   <name>yarn.nodemanager.resource-type.resource1</name>
+   <value>5T</value>
+ </property>
+
+ <property>
+   <name>yarn.nodemanager.resource-type.resource2</name>
+   <value>2M</value>
+ </property>
+
+ <property>
+   <name>yarn.nodemanager.resource-type.yarn.io/gpu</name>
+   <value>1</value>
+ </property>
+</configuration>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java

@@ -33,6 +33,7 @@ import static org.mockito.Mockito.when;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.InetSocketAddress;
 import java.security.AccessControlException;
 import java.security.PrivilegedExceptionAction;
@@ -49,6 +50,7 @@ import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CyclicBarrier;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -157,7 +159,9 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.UTCClock;
+import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Assert;
 import org.junit.Assume;
@@ -2243,4 +2247,88 @@ public class TestClientRMService {
         rmService.getApplications(request).getApplicationList().size());
     rmService.setDisplayPerUserApps(false);
   }
+
+  @Test
+  public void testRegisterNMWithDiffUnits() throws Exception {
+    ResourceUtils.resetResourceTypes();
+    Configuration yarnConf = new YarnConfiguration();
+    String resourceTypesFile = "resource-types-4.xml";
+    InputStream source =
+        yarnConf.getClassLoader().getResourceAsStream(resourceTypesFile);
+    File dest = new File(yarnConf.getClassLoader().
+        getResource(".").getPath(), "resource-types.xml");
+    FileUtils.copyInputStreamToFile(source, dest);
+    ResourceUtils.getResourceTypes();
+
+    yarnConf.setClass(
+        CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+        DominantResourceCalculator.class, ResourceCalculator.class);
+
+    MockRM rm = new MockRM(yarnConf) {
+      protected ClientRMService createClientRMService() {
+        return new ClientRMService(this.rmContext, scheduler,
+          this.rmAppManager, this.applicationACLsManager, this.queueACLsManager,
+          this.getRMContext().getRMDelegationTokenSecretManager());
+      };
+    };
+    rm.start();
+
+    Resource resource = BuilderUtils.newResource(1024, 1);
+    resource.setResourceInformation("memory-mb",
+        ResourceInformation.newInstance("memory-mb", "G", 1024));
+    resource.setResourceInformation("resource1",
+        ResourceInformation.newInstance("resource1", "T", 1));
+    resource.setResourceInformation("resource2",
+        ResourceInformation.newInstance("resource2", "M", 1));
+
+    MockNM node = rm.registerNode("host1:1234", resource);
+    node.nodeHeartbeat(true);
+
+    // Create a client.
+    Configuration conf = new Configuration();
+    YarnRPC rpc = YarnRPC.create(conf);
+    InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
+    LOG.info("Connecting to ResourceManager at " + rmAddress);
+    ApplicationClientProtocol client =
+        (ApplicationClientProtocol) rpc
+          .getProxy(ApplicationClientProtocol.class, rmAddress, conf);
+
+    // Make call
+    GetClusterNodesRequest request =
+        GetClusterNodesRequest.newInstance(EnumSet.of(NodeState.RUNNING));
+    List<NodeReport> nodeReports =
+        client.getClusterNodes(request).getNodeReports();
+    Assert.assertEquals(1, nodeReports.size());
+    Assert.assertNotSame("Node is expected to be healthy!", NodeState.UNHEALTHY,
+        nodeReports.get(0).getNodeState());
+    Assert.assertEquals(1, nodeReports.size());
+
+    //Resource 'resource1' has been passed as 1T while registering NM.
+    //1T should be converted to 1000G
+    Assert.assertEquals("G", nodeReports.get(0).getCapability().
+        getResourceInformation("resource1").getUnits());
+    Assert.assertEquals(1000, nodeReports.get(0).getCapability().
+        getResourceInformation("resource1").getValue());
+
+    //Resource 'resource2' has been passed as 1M while registering NM
+    //1M should be converted to 1000000000M
+    Assert.assertEquals("m", nodeReports.get(0).getCapability().
+        getResourceInformation("resource2").getUnits());
+    Assert.assertEquals(1000000000, nodeReports.get(0).getCapability().
+        getResourceInformation("resource2").getValue());
+
+    //Resource 'memory-mb' has been passed as 1024G while registering NM
+    //1024G should be converted to 976562Mi
+    Assert.assertEquals("Mi", nodeReports.get(0).getCapability().
+        getResourceInformation("memory-mb").getUnits());
+    Assert.assertEquals(976562, nodeReports.get(0).getCapability().
+        getResourceInformation("memory-mb").getValue());
+
+    rpc.stopProxy(client, conf);
+    rm.close();
+
+    if (dest.exists()) {
+      dest.delete();
+    }
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java

@@ -414,7 +414,7 @@ public class TestFairSchedulerConfiguration {
           calculator.normalize(customResource(10000L, ""), min, max, increment)
             .getResourceInformation(A_CUSTOM_RESOURCE));
       assertEquals(customResourceInformation(20000L, ""),
-          calculator.normalize(customResource(10001L, ""), min, max, increment)
+          calculator.normalize(customResource(19999L, ""), min, max, increment)
             .getResourceInformation(A_CUSTOM_RESOURCE));
       assertEquals(customResourceInformation(10L, "k"),
           calculator.normalize(customResource(9L, "k"), min, max, increment)