YARN-5707. Add manager class for resource profiles. Contributed by Varun Vasudev.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -851,6 +851,29 @@ public class YarnConfiguration extends Configuration {
    */
   public static final String RM_PROXY_USER_PREFIX = RM_PREFIX + "proxyuser.";
 
+  /**
+   * Enable/disable resource profiles.
+   */
+  @Public
+  @Unstable
+  public static final String RM_RESOURCE_PROFILES_ENABLED =
+      RM_PREFIX + "resource-profiles.enabled";
+  @Public
+  @Unstable
+  public static final boolean DEFAULT_RM_RESOURCE_PROFILES_ENABLED = false;
+
+  /**
+   * File containing resource profiles.
+   */
+  @Public
+  @Unstable
+  public static final String RM_RESOURCE_PROFILES_SOURCE_FILE =
+      RM_PREFIX + "resource-profiles.source-file";
+  @Public
+  @Unstable
+  public static final String DEFAULT_RM_RESOURCE_PROFILES_SOURCE_FILE =
+      "resource-profiles.json";
+
   /**
    * Timeout in seconds for YARN node graceful decommission.
    * This is the maximal time to wait for running containers and applications

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java

@@ -84,6 +84,10 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
     // Used as Java command line properties, not XML
     configurationPrefixToSkipCompare.add("yarn.app.container");
 
+    // Ignore default file name for resource profiles
+    configurationPropsToSkipCompare
+        .add(YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_SOURCE_FILE);
+
     // Ignore NodeManager "work in progress" variables
     configurationPrefixToSkipCompare
         .add(YarnConfiguration.NM_NETWORK_RESOURCE_ENABLED);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -979,6 +979,22 @@
     <value>600000</value>
   </property>
 
+  <property>
+    <description>
+    Flag to enable/disable resource profiles
+    </description>
+    <name>yarn.resourcemanager.resource-profiles.enabled</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <description>
+    If resource profiles is enabled, source file for the profiles
+    </description>
+    <name>yarn.resourcemanager.resource-profiles.source-file</name>
+    <value>resource-profiles.json</value>
+  </property>
+
   <!-- Node Manager Configuration -->
 
   <property>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml

@@ -348,6 +348,11 @@
             <exclude>src/test/resources/delete-reservation.json</exclude>
             <exclude>src/test/resources/update-reservation.json</exclude>
             <exclude>src/test/resources/invariants.txt</exclude>
+            <exclude>src/test/resources/profiles/sample-profiles-1.json</exclude>
+            <exclude>src/test/resources/profiles/sample-profiles-2.json</exclude>
+            <exclude>src/test/resources/profiles/illegal-profiles-1.json</exclude>
+            <exclude>src/test/resources/profiles/illegal-profiles-2.json</exclude>
+            <exclude>src/test/resources/profiles/illegal-profiles-3.json</exclude>
           </excludes>
         </configuration>
       </plugin>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java

@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.resource;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Interface for the resource profiles manager. Provides an interface to get
+ * the list of available profiles and some helper functions.
+ */
+public interface ResourceProfilesManager {
+
+  void init(Configuration config) throws IOException;
+
+  Resource getProfile(String profile);
+
+  Map<String, Resource> getResourceProfiles();
+
+  void reloadProfiles() throws IOException;
+
+  Resource getDefaultProfile();
+
+  Resource getMinimumProfile();
+
+  Resource getMaximumProfile();
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java

@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.resource;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
+
+  private static final Log LOG =
+      LogFactory.getLog(ResourceProfilesManagerImpl.class);
+
+  private final Map<String, Resource> profiles = new ConcurrentHashMap<>();
+  private Configuration conf;
+
+  private static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
+  private static final String VCORES = ResourceInformation.VCORES.getName();
+
+  private static final String DEFAULT_PROFILE = "default";
+  private static final String MINIMUM_PROFILE = "minimum";
+  private static final String MAXIMUM_PROFILE = "maximum";
+
+  private static final String[] MANDATORY_PROFILES =
+      { DEFAULT_PROFILE, MINIMUM_PROFILE, MAXIMUM_PROFILE };
+
+  public void init(Configuration config) throws IOException {
+    conf = config;
+    loadProfiles();
+  }
+
+  private void loadProfiles() throws IOException {
+    boolean profilesEnabled =
+        conf.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
+            YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
+    if (!profilesEnabled) {
+      return;
+    }
+    String sourceFile =
+        conf.get(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE,
+            YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_SOURCE_FILE);
+    String resourcesFile = sourceFile;
+    ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+    if (classLoader == null) {
+      classLoader = ResourceProfilesManagerImpl.class.getClassLoader();
+    }
+    if (classLoader != null) {
+      URL tmp = classLoader.getResource(sourceFile);
+      if (tmp != null) {
+        resourcesFile = tmp.getPath();
+      }
+    }
+    ObjectMapper mapper = new ObjectMapper();
+    Map data = mapper.readValue(new File(resourcesFile), Map.class);
+    Iterator iterator = data.entrySet().iterator();
+    while (iterator.hasNext()) {
+      Map.Entry entry = (Map.Entry) iterator.next();
+      String key = entry.getKey().toString();
+      if (entry.getValue() instanceof Map) {
+        Map value = (Map) entry.getValue();
+        // ensure memory and vcores are specified
+        if (!value.containsKey(MEMORY) || !value.containsKey(VCORES)) {
+          throw new IOException(
+              "Illegal resource profile definition; profile '" + key
+                  + "' must contain '" + MEMORY + "' and '" + VCORES + "'");
+        }
+        Resource resource = parseResource(key, value);
+        profiles.put(key, resource);
+        LOG.info("Added profile '" + key + "' with resources " + resource);
+      }
+    }
+    // check to make sure mandatory profiles are present
+    for (String profile : MANDATORY_PROFILES) {
+      if (!profiles.containsKey(profile)) {
+        throw new IOException(
+            "Mandatory profile missing '" + profile + "' missing. "
+                + Arrays.toString(MANDATORY_PROFILES) + " must be present");
+      }
+    }
+    LOG.info("Loaded profiles " + profiles.keySet());
+  }
+
+  private Resource parseResource(String key, Map value) throws IOException {
+    Resource resource = Resource.newInstance(0, 0);
+    Iterator iterator = value.entrySet().iterator();
+    Map<String, ResourceInformation> resourceTypes =
+        ResourceUtils.getResourceTypes();
+    while (iterator.hasNext()) {
+      Map.Entry resourceEntry = (Map.Entry) iterator.next();
+      String resourceName = resourceEntry.getKey().toString();
+      ResourceInformation resourceValue =
+          fromString(resourceName, resourceEntry.getValue().toString());
+      if (resourceName.equals(MEMORY)) {
+        resource.setMemorySize(resourceValue.getValue());
+        continue;
+      }
+      if (resourceName.equals(VCORES)) {
+        resource.setVirtualCores(resourceValue.getValue().intValue());
+        continue;
+      }
+      if (resourceTypes.containsKey(resourceName)) {
+        resource.setResourceInformation(resourceName, resourceValue);
+      } else {
+        throw new IOException("Unrecognized resource type '" + resourceName
+            + "'. Recognized resource types are '" + resourceTypes.keySet()
+            + "'");
+      }
+    }
+    return resource;
+  }
+
+  public Resource getProfile(String profile) {
+    return Resources.clone(profiles.get(profile));
+  }
+
+  public Map<String, Resource> getResourceProfiles() {
+    return Collections.unmodifiableMap(profiles);
+  }
+
+  @VisibleForTesting
+  public void reloadProfiles() throws IOException {
+    profiles.clear();
+    loadProfiles();
+  }
+
+  public Resource getDefaultProfile() {
+    return getProfile(DEFAULT_PROFILE);
+  }
+
+  public Resource getMinimumProfile() {
+    return getProfile(MINIMUM_PROFILE);
+  }
+
+  public Resource getMaximumProfile() {
+    return getProfile(MAXIMUM_PROFILE);
+  }
+
+  private ResourceInformation fromString(String name, String value) {
+    String units = ResourceUtils.getUnits(value);
+    Long resourceValue =
+        Long.valueOf(value.substring(0, value.length() - units.length()));
+    return ResourceInformation.newInstance(name, units, resourceValue);
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java

@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.resource;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class TestResourceProfiles {
+
+  @Test
+  public void testProfilesEnabled() throws Exception {
+    ResourceProfilesManager manager = new ResourceProfilesManagerImpl();
+    Configuration conf = new Configuration();
+    // be default resource profiles should not be enabled
+    manager.init(conf);
+    Map<String, Resource> profiles = manager.getResourceProfiles();
+    Assert.assertTrue(profiles.isEmpty());
+    conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
+    try {
+      manager.init(conf);
+      Assert.fail(
+          "Exception should be thrown due to missing resource profiles file");
+    } catch (IOException ie) {
+    }
+    conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE,
+        "profiles/sample-profiles-1.json");
+    manager.init(conf);
+  }
+
+  @Test
+  public void testLoadProfiles() throws Exception {
+    ResourceProfilesManager manager = new ResourceProfilesManagerImpl();
+    Configuration conf = new Configuration();
+    conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
+    conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE,
+        "profiles/sample-profiles-1.json");
+    manager.init(conf);
+    Map<String, Resource> profiles = manager.getResourceProfiles();
+    Map<String, Resource> expected = new HashMap<>();
+    expected.put("minimum", Resource.newInstance(1024, 1));
+    expected.put("default", Resource.newInstance(2048, 2));
+    expected.put("maximum", Resource.newInstance(4096, 4));
+
+    for (Map.Entry<String, Resource> entry : expected.entrySet()) {
+      String profile = entry.getKey();
+      Resource res = entry.getValue();
+      Assert.assertTrue("Mandatory profile '" + profile + "' missing",
+          profiles.containsKey(profile));
+      Assert.assertEquals("Profile " + profile + "' resources don't match", res,
+          manager.getProfile(profile));
+    }
+  }
+
+  @Test
+  public void testLoadProfilesMissingMandatoryProfile() throws Exception {
+
+    Configuration conf = new Configuration();
+    conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
+
+    String[] badProfiles = { "profiles/illegal-profiles-1.json",
+        "profiles/illegal-profiles-2.json",
+        "profiles/illegal-profiles-3.json" };
+    for (String file : badProfiles) {
+      ResourceProfilesManager manager = new ResourceProfilesManagerImpl();
+      conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE, file);
+      try {
+        manager.init(conf);
+        Assert.fail("Bad profile '" + file + "' is not valid");
+      } catch (IOException ie) {
+      }
+    }
+  }
+
+  @Test
+  public void testGetProfile() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
+    ResourceProfilesManager manager = new ResourceProfilesManagerImpl();
+    conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE,
+        "profiles/sample-profiles-2.json");
+    manager.init(conf);
+    Map<String, Resource> expected = new HashMap<>();
+    expected.put("minimum", Resource.newInstance(1024, 1));
+    expected.put("default", Resource.newInstance(2048, 2));
+    expected.put("maximum", Resource.newInstance(4096, 4));
+    expected.put("small", Resource.newInstance(1024, 1));
+    expected.put("medium", Resource.newInstance(2048, 1));
+    expected.put("large", Resource.newInstance(4096, 4));
+
+    for (Map.Entry<String, Resource> entry : expected.entrySet()) {
+      String profile = entry.getKey();
+      Resource res = entry.getValue();
+      Assert.assertEquals("Profile " + profile + "' resources don't match", res,
+          manager.getProfile(profile));
+    }
+  }
+
+  @Test
+  public void testGetMandatoryProfiles() throws Exception {
+    ResourceProfilesManager manager = new ResourceProfilesManagerImpl();
+    Configuration conf = new Configuration();
+    conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
+    conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE,
+        "profiles/sample-profiles-1.json");
+    manager.init(conf);
+    Map<String, Resource> expected = new HashMap<>();
+    expected.put("minimum", Resource.newInstance(1024, 1));
+    expected.put("default", Resource.newInstance(2048, 2));
+    expected.put("maximum", Resource.newInstance(4096, 4));
+
+    Assert.assertEquals("Profile 'minimum' resources don't match",
+        expected.get("minimum"), manager.getMinimumProfile());
+    Assert.assertEquals("Profile 'default' resources don't match",
+        expected.get("default"), manager.getDefaultProfile());
+    Assert.assertEquals("Profile 'maximum' resources don't match",
+        expected.get("maximum"), manager.getMaximumProfile());
+
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/illegal-profiles-1.json

@@ -0,0 +1,10 @@
+{
+    "minimum": {
+        "memoryMB" : 1024,
+        "vcores" : 1
+    },
+    "default" : {
+        "memoryMB" : 2048,
+        "vcores" : 2
+    },
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/illegal-profiles-2.json

@@ -0,0 +1,10 @@
+{
+    "minimum": {
+        "memoryMB" : 1024,
+        "vcores" : 1
+    },
+    "maximum" : {
+        "memoryMB": 4096,
+        "vcores" : 4
+    }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/illegal-profiles-3.json

@@ -0,0 +1,10 @@
+{
+    "default" : {
+        "memoryMB" : 2048,
+        "vcores" : 2
+    },
+    "maximum" : {
+        "memoryMB": 4096,
+        "vcores" : 4
+    }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-1.json

@@ -0,0 +1,14 @@
+{
+    "minimum": {
+        "memory-mb" : 1024,
+        "vcores" : 1
+    },
+    "default" : {
+        "memory-mb" : 2048,
+        "vcores" : 2
+    },
+    "maximum" : {
+        "memory-mb": 4096,
+        "vcores" : 4
+    }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-2.json

@@ -0,0 +1,26 @@
+{
+    "minimum": {
+        "memory-mb" : 1024,
+        "vcores" : 1
+    },
+    "default" : {
+        "memory-mb" : 2048,
+        "vcores" : 2
+    },
+    "maximum" : {
+        "memory-mb": 4096,
+        "vcores" : 4
+    },
+    "small" : {
+        "memory-mb": 1024,
+        "vcores": 1
+    },
+    "medium" : {
+        "memory-mb": 2048,
+        "vcores": 1
+    },
+    "large": {
+        "memory-mb" : 4096,
+        "vcores" : 4
+    }
+}