YARN-4715. Add support to read resource types from a config file. Contributed by Varun Vasudev.

(cherry picked from commit ceb12c59a34677518a198e19acb4e29cdfc8a0e1)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java

@@ -101,15 +101,6 @@ public abstract class Resource implements Comparable<Resource> {
     return new SimpleResource(memory, vCores);
   }
 
-  @Public
-  @Stable
-  public static Resource newInstance(
-      Map<String, ResourceInformation> resources) {
-    Resource resource = Records.newRecord(Resource.class);
-    resource.setResources(resources);
-    return resource;
-  }
-
   /**
    * This method is DEPRECATED:
    * Use {@link Resource#getMemorySize()} instead
@@ -233,15 +224,6 @@ public abstract class Resource implements Comparable<Resource> {
   @Evolving
   public abstract Long getResourceValue(String resource) throws YarnException;
 
-  /**
-   * Set the resources to the map specified.
-   *
-   * @param resources Desired resources
-   */
-  @Public
-  @Evolving
-  public abstract void setResources(Map<String, ResourceInformation> resources);
-
   /**
    * Set the ResourceInformation object for a particular resource.
    *
@@ -276,8 +258,8 @@ public abstract class Resource implements Comparable<Resource> {
     result = prime * result + getVirtualCores();
     for (Map.Entry<String, ResourceInformation> entry : getResources()
         .entrySet()) {
-      if (entry.getKey().equals(ResourceInformation.MEMORY.getName()) || entry
-          .getKey().equals(ResourceInformation.VCORES.getName())) {
+      if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
+          || entry.getKey().equals(ResourceInformation.VCORES.getName())) {
         continue;
       }
       result = prime * result + entry.getValue().hashCode();
@@ -320,7 +302,7 @@ public abstract class Resource implements Comparable<Resource> {
         .append(getVirtualCores());
     for (Map.Entry<String, ResourceInformation> entry : getResources()
         .entrySet()) {
-      if (entry.getKey().equals(ResourceInformation.MEMORY.getName())
+      if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
           && entry.getValue().getUnits()
           .equals(ResourceInformation.MEMORY_MB.getUnits())) {
         continue;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java

@@ -32,11 +32,9 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
   private ResourceTypes resourceType;
   private Long value;
 
-  private static final String MEMORY_URI = "yarn.io/memory";
-  private static final String VCORES_URI = "yarn.io/vcores";
+  private static final String MEMORY_URI = "memory-mb";
+  private static final String VCORES_URI = "vcores";
 
-  public static final ResourceInformation MEMORY =
-      ResourceInformation.newInstance(MEMORY_URI);
   public static final ResourceInformation MEMORY_MB =
       ResourceInformation.newInstance(MEMORY_URI, "M");
   public static final ResourceInformation VCORES =
@@ -77,7 +75,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
   public void setUnits(String rUnits) {
     if (!UnitsConversionUtil.KNOWN_UNITS.contains(rUnits)) {
       throw new IllegalArgumentException(
-          "Unknown unit '" + units + "'. Known units are "
+          "Unknown unit '" + rUnits + "'. Known units are "
               + UnitsConversionUtil.KNOWN_UNITS);
     }
     this.units = rUnits;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -60,9 +60,14 @@ public class YarnConfiguration extends Configuration {
   @Private
   public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml";
 
+  @Private
+  public static final String RESOURCE_TYPES_CONFIGURATION_FILE =
+      "resource-types.xml";
+
   @Private
   public static final List<String> RM_CONFIGURATION_FILES =
       Collections.unmodifiableList(Arrays.asList(
+          RESOURCE_TYPES_CONFIGURATION_FILE,
           DR_CONFIGURATION_FILE,
           CS_CONFIGURATION_FILE,
           HADOOP_POLICY_CONFIGURATION_FILE,
@@ -106,6 +111,13 @@ public class YarnConfiguration extends Configuration {
 
   public static final String YARN_PREFIX = "yarn.";
 
+  /////////////////////////////
+  // Scheduler resource types configs
+  ////////////////////////////
+
+  public static final String RESOURCE_TYPES =
+      YarnConfiguration.YARN_PREFIX + "resource-types";
+
   /** Delay before deleting resource to ease debugging of NM issues */
   public static final String DEBUG_NM_DELETE_DELAY_SEC =
     YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ResourceNotFoundException.java

@@ -27,7 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability;
  */
 @InterfaceAudience.Public
 @InterfaceStability.Unstable
-public class ResourceNotFoundException extends YarnException {
+public class ResourceNotFoundException extends YarnRuntimeException {
 
   private static final long serialVersionUID = 10081982L;
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml

@@ -213,6 +213,14 @@
         <filtering>true</filtering>
       </resource>
     </resources>
+    <testResources>
+      <testResource>
+        <directory>${project.basedir}/src/test/resources</directory>
+      </testResource>
+      <testResource>
+        <directory>${project.basedir}/src/test/resources/resource-types</directory>
+      </testResource>
+    </testResources>
     <plugins>
      <plugin>
         <groupId>org.apache.rat</groupId>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java

@@ -25,10 +25,11 @@ import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProtoOrBuilder;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceInformationProto;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.UnitsConversionUtil;
 
 import java.util.*;
@@ -36,6 +37,7 @@ import java.util.*;
 @Private
 @Unstable
 public class ResourcePBImpl extends Resource {
+
   ResourceProto proto = ResourceProto.getDefaultInstance();
   ResourceProto.Builder builder = null;
   boolean viaProto = false;
@@ -89,16 +91,11 @@ public class ResourcePBImpl extends Resource {
 
   @Override
   public long getMemorySize() {
-    try {
-      ResourceInformation ri =
-          this.getResourceInformation(ResourceInformation.MEMORY.getName());
-      return (int) UnitsConversionUtil
-          .convert(ri.getUnits(), "M", ri.getValue()).longValue();
-    } catch (YarnException ye) {
-      // memory should always be present
-      initResourcesMap();
-      return 0;
-    }
+    // memory should always be present
+    initResourcesMap();
+    ResourceInformation ri =
+        this.getResourceInformation(ResourceInformation.MEMORY_MB.getName());
+    return UnitsConversionUtil.convert(ri.getUnits(), "M", ri.getValue());
   }
 
   @Override
@@ -117,14 +114,10 @@ public class ResourcePBImpl extends Resource {
 
   @Override
   public int getVirtualCores() {
-    try {
-      return (int) this.getResourceValue(ResourceInformation.VCORES.getName())
-          .longValue();
-    } catch (YarnException ye) {
-      // vcores should always be present
-      initResourcesMap();
-      return 0;
-    }
+    // vcores should always be present
+    initResourcesMap();
+    return this.getResourceValue(ResourceInformation.VCORES.getName())
+        .intValue();
   }
 
   @Override
@@ -163,21 +156,6 @@ public class ResourcePBImpl extends Resource {
     }
   }
 
-  @Override
-  public void setResources(Map<String, ResourceInformation> resources) {
-    maybeInitBuilder();
-    if (resources == null || resources.isEmpty()) {
-      builder.clearResourceValueMap();
-    } else {
-      for (Map.Entry<String, ResourceInformation> entry : resources.entrySet()) {
-        if (!entry.getKey().equals(entry.getValue().getName())) {
-          entry.getValue().setName(entry.getKey());
-        }
-      }
-    }
-    this.resources = resources;
-  }
-
   @Override
   public void setResourceInformation(String resource,
       ResourceInformation resourceInformation) {
@@ -216,8 +194,7 @@ public class ResourcePBImpl extends Resource {
   }
 
   @Override
-  public ResourceInformation getResourceInformation(String resource)
-      throws YarnException {
+  public ResourceInformation getResourceInformation(String resource) {
     initResources();
     if (this.resources.containsKey(resource)) {
       return this.resources.get(resource);
@@ -226,7 +203,7 @@ public class ResourcePBImpl extends Resource {
   }
 
   @Override
-  public Long getResourceValue(String resource) throws YarnException {
+  public Long getResourceValue(String resource) {
     initResources();
     if (this.resources.containsKey(resource)) {
       return this.resources.get(resource).getValue();
@@ -237,18 +214,15 @@ public class ResourcePBImpl extends Resource {
   private void initResourcesMap() {
     if (resources == null) {
       resources = new HashMap<>();
-    }
-    ResourceInformation ri;
-    if (!resources.containsKey(ResourceInformation.MEMORY.getName())) {
-      ri = ResourceInformation
-          .newInstance(ResourceInformation.MEMORY_MB.getName(),
-              ResourceInformation.MEMORY_MB.getUnits());
-      this.resources.put(ResourceInformation.MEMORY.getName(), ri);
-    }
-    if (!resources.containsKey(ResourceInformation.VCORES.getName())) {
-      ri =
-          ResourceInformation.newInstance(ResourceInformation.VCORES.getName());
-      this.resources.put(ResourceInformation.VCORES.getName(), ri);
+      Map<String, ResourceInformation> types = ResourceUtils.getResourceTypes();
+      if (types == null) {
+        throw new YarnRuntimeException(
+            "Got null return value from ResourceUtils.getResourceTypes()");
+      }
+      for (Map.Entry<String, ResourceInformation> entry : types.entrySet()) {
+        resources.put(entry.getKey(),
+            ResourceInformation.newInstance(entry.getValue()));
+      }
     }
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java

@@ -26,7 +26,6 @@ import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.UnitsConversionUtil;
 
-import java.util.HashSet;
 import java.util.Set;
 
 /**
@@ -60,9 +59,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
   private Set<String> resourceNames;
 
   public DominantResourceCalculator() {
-    resourceNames = new HashSet<>();
-    resourceNames.add(ResourceInformation.MEMORY.getName());
-    resourceNames.add(ResourceInformation.VCORES.getName());
+    resourceNames = ResourceUtils.getResourceTypes().keySet();
   }
 
   /**

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java

@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util.resource;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.conf.ConfigurationProvider;
+import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Helper class to read the resource-types to be supported by the system.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class ResourceUtils {
+
+  public static final String UNITS = ".units";
+  public static final String TYPE = ".type";
+
+  private static final Set<String> DISALLOWED_NAMES = new HashSet<>();
+  static {
+    DISALLOWED_NAMES.add("memory");
+    DISALLOWED_NAMES.add(ResourceInformation.MEMORY_MB.getName());
+    DISALLOWED_NAMES.add(ResourceInformation.VCORES.getName());
+  }
+
+  private static volatile Object lock;
+  private static Map<String, ResourceInformation> readOnlyResources;
+
+  static final Log LOG = LogFactory.getLog(ResourceUtils.class);
+
+  private ResourceUtils() {
+  }
+
+  private static void checkMandatatoryResources(
+      Map<String, ResourceInformation> resourceInformationMap)
+      throws YarnRuntimeException {
+    String memory = ResourceInformation.MEMORY_MB.getName();
+    String vcores = ResourceInformation.VCORES.getName();
+    if (resourceInformationMap.containsKey(memory)) {
+      ResourceInformation memInfo = resourceInformationMap.get(memory);
+      String memUnits = ResourceInformation.MEMORY_MB.getUnits();
+      ResourceTypes memType = ResourceInformation.MEMORY_MB.getResourceType();
+      if (!memInfo.getUnits().equals(memUnits) || !memInfo.getResourceType()
+          .equals(memType)) {
+        throw new YarnRuntimeException(
+            "Attempt to re-define mandatory resource 'memory-mb'. It can only"
+                + " be of type 'COUNTABLE' and have units 'M'.");
+      }
+    }
+
+    if (resourceInformationMap.containsKey(vcores)) {
+      ResourceInformation vcoreInfo = resourceInformationMap.get(vcores);
+      String vcoreUnits = ResourceInformation.VCORES.getUnits();
+      ResourceTypes vcoreType = ResourceInformation.VCORES.getResourceType();
+      if (!vcoreInfo.getUnits().equals(vcoreUnits) || !vcoreInfo
+          .getResourceType().equals(vcoreType)) {
+        throw new YarnRuntimeException(
+            "Attempt to re-define mandatory resource 'vcores'. It can only be"
+                + " of type 'COUNTABLE' and have units ''(no units).");
+      }
+    }
+  }
+
+  private static void addManadtoryResources(
+      Map<String, ResourceInformation> res) {
+    ResourceInformation ri;
+    if (!res.containsKey(ResourceInformation.MEMORY_MB.getName())) {
+      LOG.info("Adding resource type - name = " + ResourceInformation.MEMORY_MB
+          .getName() + ", units = " + ResourceInformation.MEMORY_MB.getUnits()
+          + ", type = " + ResourceTypes.COUNTABLE);
+      ri = ResourceInformation
+          .newInstance(ResourceInformation.MEMORY_MB.getName(),
+              ResourceInformation.MEMORY_MB.getUnits());
+      res.put(ResourceInformation.MEMORY_MB.getName(), ri);
+    }
+    if (!res.containsKey(ResourceInformation.VCORES.getName())) {
+      LOG.info("Adding resource type - name = " + ResourceInformation.VCORES
+          .getName() + ", units = , type = " + ResourceTypes.COUNTABLE);
+      ri =
+          ResourceInformation.newInstance(ResourceInformation.VCORES.getName());
+      res.put(ResourceInformation.VCORES.getName(), ri);
+    }
+  }
+
+  @VisibleForTesting
+  static void initializeResourcesMap(Configuration conf,
+      Map<String, ResourceInformation> resourceInformationMap) {
+
+    String[] resourceNames = conf.getStrings(YarnConfiguration.RESOURCE_TYPES);
+    if (resourceNames != null && resourceNames.length != 0) {
+      for (String resourceName : resourceNames) {
+        String resourceUnits = conf.get(
+            YarnConfiguration.RESOURCE_TYPES + "." + resourceName + UNITS, "");
+        String resourceTypeName = conf.get(
+            YarnConfiguration.RESOURCE_TYPES + "." + resourceName + TYPE,
+            ResourceTypes.COUNTABLE.toString());
+        if (resourceName == null || resourceName.isEmpty()
+            || resourceUnits == null || resourceTypeName == null) {
+          throw new YarnRuntimeException(
+              "Incomplete configuration for resource type '" + resourceName
+                  + "'. One of name, units or type is configured incorrectly.");
+        }
+        if (DISALLOWED_NAMES.contains(resourceName)) {
+          throw new YarnRuntimeException(
+              "Resource type cannot be named '" + resourceName
+                  + "'. That name is disallowed.");
+        }
+        ResourceTypes resourceType = ResourceTypes.valueOf(resourceTypeName);
+        LOG.info("Adding resource type - name = " + resourceName + ", units = "
+            + resourceUnits + ", type = " + resourceTypeName);
+        if (resourceInformationMap.containsKey(resourceName)) {
+          throw new YarnRuntimeException(
+              "Error in config, key '" + resourceName + "' specified twice");
+        }
+        resourceInformationMap.put(resourceName, ResourceInformation
+            .newInstance(resourceName, resourceUnits, 0L, resourceType));
+      }
+    }
+    checkMandatatoryResources(resourceInformationMap);
+    addManadtoryResources(resourceInformationMap);
+    readOnlyResources = Collections.unmodifiableMap(resourceInformationMap);
+  }
+
+  /**
+   * Get the resource types to be supported by the system.
+   * @return A map of the resource name to a ResouceInformation object
+   *         which contains details such as the unit.
+   */
+  public static Map<String, ResourceInformation> getResourceTypes() {
+    return getResourceTypes(null,
+        YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+  }
+
+  private static Map<String, ResourceInformation> getResourceTypes(
+      Configuration conf, String resourceFile) {
+    if (lock == null) {
+      synchronized (ResourceUtils.class) {
+        if (lock == null) {
+          synchronized (ResourceUtils.class) {
+            lock = new Object();
+            Map<String, ResourceInformation> resources = new HashMap<>();
+            if (conf == null) {
+              conf = new YarnConfiguration();
+            }
+            try {
+              InputStream ris = getConfInputStream(resourceFile, conf);
+              LOG.debug("Found " + resourceFile + ", adding to configuration");
+              conf.addResource(ris);
+              initializeResourcesMap(conf, resources);
+              return resources;
+            } catch (FileNotFoundException fe) {
+              LOG.info("Unable to find '" + resourceFile
+                  + "'. Falling back to memory and vcores as resources", fe);
+              initializeResourcesMap(conf, resources);
+            } catch (IOException ie) {
+              LOG.fatal(
+                  "Exception trying to read resource types configuration '"
+                      + resourceFile + "'.", ie);
+              throw new YarnRuntimeException(ie);
+            } catch (YarnException ye) {
+              LOG.fatal(
+                  "YARN Exception trying to read resource types configuration '"
+                      + resourceFile + "'.", ye);
+              throw new YarnRuntimeException(ye);
+            }
+          }
+        }
+      }
+    }
+    return readOnlyResources;
+  }
+
+  static InputStream getConfInputStream(String resourceFile, Configuration conf)
+      throws IOException, YarnException {
+
+    ConfigurationProvider provider =
+        ConfigurationProviderFactory.getConfigurationProvider(conf);
+    InputStream ris = provider.getConfigurationInputStream(conf, resourceFile);
+    if (ris == null) {
+      if (conf.getResource(resourceFile) == null) {
+        throw new FileNotFoundException("Unable to find " + resourceFile);
+      }
+      throw new IOException(
+          "Unable to open resource types file '" + resourceFile
+              + "'. Using provider " + provider);
+    }
+    return ris;
+  }
+
+  @VisibleForTesting
+  static void resetResourceTypes() {
+    lock = null;
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java

@@ -119,11 +119,6 @@ public class Resources {
       throw new YarnException("" + resource + " not found");
     }
 
-    @Override
-    public void setResources(Map<String, ResourceInformation> resources) {
-      throw new RuntimeException(name + " cannot be modified!");
-    }
-
     @Override
     public void setResourceInformation(String resource,
         ResourceInformation resourceInformation)
@@ -143,11 +138,14 @@ public class Resources {
       // needs to be Integer.MAX_VALUE
       int max = resourceValue > Integer.MAX_VALUE ? Integer.MAX_VALUE :
           resourceValue.intValue();
-      tmp.put(ResourceInformation.MEMORY.getName(), ResourceInformation
-          .newInstance(ResourceInformation.MEMORY.getName(),
-              ResourceInformation.MEMORY_MB.getUnits(), (long) max));
-      tmp.put(ResourceInformation.VCORES.getName(), ResourceInformation
-          .newInstance(ResourceInformation.VCORES.getName(), (long) max));
+      Map<String, ResourceInformation> types = ResourceUtils.getResourceTypes();
+      if (types != null) {
+        for (Map.Entry<String, ResourceInformation> entry : types.entrySet()) {
+          tmp.put(entry.getKey(),
+              ResourceInformation.newInstance(entry.getValue()));
+          tmp.get(entry.getKey()).setValue((long) max);
+        }
+      }
       return tmp;
     }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -3404,4 +3404,14 @@
     <name>yarn.scheduler.configuration.zk-store.parent-path</name>
     <value>/confstore</value>
   </property>
+
+  <!-- resource types configuration -->
+  <property>
+    <name>yarn.resource-types</name>
+    <value></value>
+    <description>
+    The resource types to be used for scheduling. Use resource-types.xml
+    to specify details about the individual resource types.
+    </description>
+  </property>
 </configuration>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java

@@ -0,0 +1,248 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util.resource;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.SystemClock;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+
+public class TestResourceUtils {
+
+  static class ResourceFileInformation {
+    String filename;
+    int resourceCount;
+    Map<String, String> resourceNameUnitsMap;
+
+    public ResourceFileInformation(String name, int count) {
+      filename = name;
+      resourceCount = count;
+      resourceNameUnitsMap = new HashMap<>();
+    }
+  }
+
+  @Before
+  public void setup() {
+    ResourceUtils.resetResourceTypes();
+  }
+
+  private void testMemoryAndVcores(Map<String, ResourceInformation> res) {
+    String memory = ResourceInformation.MEMORY_MB.getName();
+    String vcores = ResourceInformation.VCORES.getName();
+    Assert.assertTrue("Resource 'memory' missing", res.containsKey(memory));
+    Assert.assertEquals("'memory' units incorrect",
+        ResourceInformation.MEMORY_MB.getUnits(), res.get(memory).getUnits());
+    Assert.assertEquals("'memory' types incorrect",
+        ResourceInformation.MEMORY_MB.getResourceType(),
+        res.get(memory).getResourceType());
+    Assert.assertTrue("Resource 'vcores' missing", res.containsKey(vcores));
+    Assert.assertEquals("'vcores' units incorrect",
+        ResourceInformation.VCORES.getUnits(), res.get(vcores).getUnits());
+    Assert.assertEquals("'vcores' type incorrect",
+        ResourceInformation.VCORES.getResourceType(),
+        res.get(vcores).getResourceType());
+  }
+
+  @Test
+  public void testGetResourceTypes() throws Exception {
+
+    Map<String, ResourceInformation> res = ResourceUtils.getResourceTypes();
+    Assert.assertEquals(2, res.size());
+    testMemoryAndVcores(res);
+  }
+
+  @Test
+  public void testGetResourceTypesConfigs() throws Exception {
+
+    Configuration conf = new YarnConfiguration();
+
+    ResourceFileInformation testFile1 =
+        new ResourceFileInformation("resource-types-1.xml", 2);
+    ResourceFileInformation testFile2 =
+        new ResourceFileInformation("resource-types-2.xml", 3);
+    testFile2.resourceNameUnitsMap.put("resource1", "G");
+    ResourceFileInformation testFile3 =
+        new ResourceFileInformation("resource-types-3.xml", 3);
+    testFile3.resourceNameUnitsMap.put("resource2", "");
+    ResourceFileInformation testFile4 =
+        new ResourceFileInformation("resource-types-4.xml", 4);
+    testFile4.resourceNameUnitsMap.put("resource1", "G");
+    testFile4.resourceNameUnitsMap.put("resource2", "m");
+
+    ResourceFileInformation[] tests =
+        { testFile1, testFile2, testFile3, testFile4 };
+    Map<String, ResourceInformation> res;
+    for (ResourceFileInformation testInformation : tests) {
+      ResourceUtils.resetResourceTypes();
+      File source = new File(
+          conf.getClassLoader().getResource(testInformation.filename)
+              .getFile());
+      File dest = new File(source.getParent(), "resource-types.xml");
+      FileUtils.copyFile(source, dest);
+      res = ResourceUtils.getResourceTypes();
+      testMemoryAndVcores(res);
+      Assert.assertEquals(testInformation.resourceCount, res.size());
+      for (Map.Entry<String, String> entry : testInformation.resourceNameUnitsMap
+          .entrySet()) {
+        String resourceName = entry.getKey();
+        Assert.assertTrue("Missing key " + resourceName,
+            res.containsKey(resourceName));
+        Assert.assertEquals(entry.getValue(), res.get(resourceName).getUnits());
+      }
+      dest.delete();
+    }
+  }
+
+  @Test
+  public void testGetResourceTypesConfigErrors() throws Exception {
+    Configuration conf = new YarnConfiguration();
+
+    String[] resourceFiles =
+        { "resource-types-error-1.xml", "resource-types-error-2.xml",
+            "resource-types-error-3.xml", "resource-types-error-4.xml" };
+    for (String resourceFile : resourceFiles) {
+      ResourceUtils.resetResourceTypes();
+      File dest = null;
+      try {
+        File source =
+            new File(conf.getClassLoader().getResource(resourceFile).getFile());
+        dest = new File(source.getParent(), "resource-types.xml");
+        FileUtils.copyFile(source, dest);
+        ResourceUtils.getResourceTypes();
+        Assert.fail("Expected error with file " + resourceFile);
+      } catch (NullPointerException ne) {
+        throw ne;
+      } catch (Exception e) {
+        if (dest != null) {
+          dest.delete();
+        }
+      }
+    }
+  }
+
+  @Test
+  public void testInitializeResourcesMap() throws Exception {
+    String[] empty = { "", "" };
+    String[] res1 = { "resource1", "m" };
+    String[] res2 = { "resource2", "G" };
+    String[][] test1 = { empty };
+    String[][] test2 = { res1 };
+    String[][] test3 = { res2 };
+    String[][] test4 = { res1, res2 };
+
+    String[][][] allTests = { test1, test2, test3, test4 };
+
+    for (String[][] test : allTests) {
+
+      Configuration conf = new YarnConfiguration();
+      String resSt = "";
+      for (String[] resources : test) {
+        resSt += (resources[0] + ",");
+      }
+      resSt = resSt.substring(0, resSt.length() - 1);
+      conf.set(YarnConfiguration.RESOURCE_TYPES, resSt);
+      for (String[] resources : test) {
+        String name =
+            YarnConfiguration.RESOURCE_TYPES + "." + resources[0] + ".units";
+        conf.set(name, resources[1]);
+      }
+      Map<String, ResourceInformation> ret = new HashMap<>();
+      ResourceUtils.initializeResourcesMap(conf, ret);
+      // for test1, 4 - length will be 1, 4
+      // for the others, len will be 3
+      int len = 3;
+      if (test == test1) {
+        len = 2;
+      } else if (test == test4) {
+        len = 4;
+      }
+
+      Assert.assertEquals(len, ret.size());
+      for (String[] resources : test) {
+        if (resources[0].length() == 0) {
+          continue;
+        }
+        Assert.assertTrue(ret.containsKey(resources[0]));
+        ResourceInformation resInfo = ret.get(resources[0]);
+        Assert.assertEquals(resources[1], resInfo.getUnits());
+        Assert.assertEquals(ResourceTypes.COUNTABLE, resInfo.getResourceType());
+      }
+      // we must always have memory and vcores with their fixed units
+      Assert.assertTrue(ret.containsKey("memory-mb"));
+      ResourceInformation memInfo = ret.get("memory-mb");
+      Assert.assertEquals("M", memInfo.getUnits());
+      Assert.assertEquals(ResourceTypes.COUNTABLE, memInfo.getResourceType());
+      Assert.assertTrue(ret.containsKey("vcores"));
+      ResourceInformation vcoresInfo = ret.get("vcores");
+      Assert.assertEquals("", vcoresInfo.getUnits());
+      Assert
+          .assertEquals(ResourceTypes.COUNTABLE, vcoresInfo.getResourceType());
+    }
+  }
+
+  @Test
+  public void testInitializeResourcesMapErrors() throws Exception {
+
+    String[] mem1 = { "memory-mb", "" };
+    String[] vcores1 = { "vcores", "M" };
+
+    String[] mem2 = { "memory-mb", "m" };
+    String[] vcores2 = { "vcores", "G" };
+
+    String[] mem3 = { "memory", "" };
+
+    String[][] test1 = { mem1, vcores1 };
+    String[][] test2 = { mem2, vcores2 };
+    String[][] test3 = { mem3 };
+
+    String[][][] allTests = { test1, test2, test3 };
+
+    for (String[][] test : allTests) {
+
+      Configuration conf = new YarnConfiguration();
+      String resSt = "";
+      for (String[] resources : test) {
+        resSt += (resources[0] + ",");
+      }
+      resSt = resSt.substring(0, resSt.length() - 1);
+      conf.set(YarnConfiguration.RESOURCE_TYPES, resSt);
+      for (String[] resources : test) {
+        String name =
+            YarnConfiguration.RESOURCE_TYPES + "." + resources[0] + ".units";
+        conf.set(name, resources[1]);
+      }
+      Map<String, ResourceInformation> ret = new HashMap<>();
+      try {
+        ResourceUtils.initializeResourcesMap(conf, ret);
+        Assert.fail("resource map initialization should fail");
+      } catch (Exception e) {
+        // do nothing
+      }
+    }
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-1.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+</configuration>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-2.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+
+ <property>
+   <name>yarn.resource-types</name>
+   <value>resource1</value>
+ </property>
+
+ <property>
+   <name>yarn.resource-types.resource1.units</name>
+   <value>G</value>
+ </property>
+
+</configuration>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-3.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+
+ <property>
+   <name>yarn.resource-types</name>
+   <value>resource2</value>
+ </property>
+
+</configuration>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+
+ <property>
+   <name>yarn.resource-types</name>
+   <value>resource1,resource2</value>
+ </property>
+
+ <property>
+   <name>yarn.resource-types.resource1.units</name>
+   <value>G</value>
+ </property>
+
+ <property>
+   <name>yarn.resource-types.resource2.units</name>
+   <value>m</value>
+ </property>
+
+</configuration>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-1.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+
+ <property>
+   <name>yarn.resource-types</name>
+   <value>memory-mb,resource1</value>
+ </property>
+
+ <property>
+   <name>yarn.resource-types.resource1.calculator-units</name>
+   <value>G</value>
+ </property>
+
+</configuration>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+
+ <property>
+   <name>yarn.resource-types</name>
+   <value>vcores,resource1</value>
+ </property>
+
+ <property>
+   <name>yarn.resource-types.resource1.calculator-units</name>
+   <value>G</value>
+ </property>
+
+</configuration>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+
+ <property>
+   <name>yarn.resource-types</name>
+   <value>vcores,resource1</value>
+ </property>
+
+ <property>
+   <name>yarn.resource-types.resource1.calculator-units</name>
+   <value>A</value>
+ </property>
+
+</configuration>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-4.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+
+ <property>
+   <name>yarn.resource-types</name>
+   <value>memory,resource1</value>
+ </property>
+
+</configuration>