svn merge -c 1364764 FIXES: MAPREDUCE-3893. allow capacity scheduler configs max-apps and max-am-pct per queue (tgraves via bobby)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1364765 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -618,6 +618,10 @@ Release 0.23.3 - UNRELEASED
     MAPREDUCE-4448. Fix NM crash during app cleanup if aggregation didn't
     init. (Jason Lowe via daryn)
 
+    MAPREDUCE-3893. allow capacity scheduler configs maximum-applications and
+    maximum-am-resource-percent configurable on a per queue basis (tgraves via
+    bobby)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java

@@ -46,13 +46,21 @@ public class CapacitySchedulerConfiguration extends Configuration {
   @Private
   public static final String DOT = ".";
   
+  @Private
+  public static final String MAXIMUM_APPLICATIONS_SUFFIX =
+    "maximum-applications";
+  
   @Private
   public static final String MAXIMUM_SYSTEM_APPLICATIONS =
-    PREFIX + "maximum-applications";
+    PREFIX + MAXIMUM_APPLICATIONS_SUFFIX;
+  
+  @Private
+  public static final String MAXIMUM_AM_RESOURCE_SUFFIX =
+    "maximum-am-resource-percent";
   
   @Private
   public static final String MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT =
-    PREFIX + "maximum-am-resource-percent";
+    PREFIX + MAXIMUM_AM_RESOURCE_SUFFIX;
   
   @Private
   public static final String QUEUES = "queues";
@@ -131,6 +139,30 @@ public class CapacitySchedulerConfiguration extends Configuration {
     return getFloat(MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 
         DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT);
   }
+
+
+  /**
+   * Get the maximum applications per queue setting.
+   * @param queue name of the queue
+   * @return setting specified or -1 if not set
+   */
+  public int getMaximumApplicationsPerQueue(String queue) {
+    int maxApplicationsPerQueue = 
+      getInt(getQueuePrefix(queue) + MAXIMUM_APPLICATIONS_SUFFIX, 
+          (int)UNDEFINED);
+    return maxApplicationsPerQueue;
+  }
+
+  /**
+   * Get the maximum am resource percent per queue setting.
+   * @param queue name of the queue
+   * @return per queue setting or defaults to the global am-resource-percent 
+   *         setting if per queue setting not present
+   */
+  public float getMaximumApplicationMasterResourcePerQueuePercent(String queue) {
+    return getFloat(getQueuePrefix(queue) + MAXIMUM_AM_RESOURCE_SUFFIX, 
+    		getMaximumApplicationMasterResourcePercent());
+  }
   
   public float getCapacity(String queue) {
     float capacity = getFloat(getQueuePrefix(queue) + CAPACITY, UNDEFINED);

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java

@@ -85,7 +85,7 @@ public class LeafQueue implements CSQueue {
   private int maxApplications;
   private int maxApplicationsPerUser;
   
-  private float maxAMResourcePercent;
+  private float maxAMResourcePerQueuePercent;
   private int maxActiveApplications; // Based on absolute max capacity
   private int maxActiveAppsUsingAbsCap; // Based on absolute capacity
   private int maxActiveApplicationsPerUser;
@@ -156,21 +156,25 @@ public class LeafQueue implements CSQueue {
     float userLimitFactor = 
       cs.getConfiguration().getUserLimitFactor(getQueuePath());
 
-    int maxSystemJobs = cs.getConfiguration().getMaximumSystemApplications();
-    int maxApplications = (int)(maxSystemJobs * absoluteCapacity);
-    int maxApplicationsPerUser = 
+    int maxApplications = cs.getConfiguration().getMaximumApplicationsPerQueue(getQueuePath());
+    if (maxApplications < 0) {
+      int maxSystemApps = cs.getConfiguration().getMaximumSystemApplications();
+      maxApplications = (int)(maxSystemApps * absoluteCapacity);
+    }
+    maxApplicationsPerUser = 
       (int)(maxApplications * (userLimit / 100.0f) * userLimitFactor);
 
-    this.maxAMResourcePercent = 
-        cs.getConfiguration().getMaximumApplicationMasterResourcePercent();
+    this.maxAMResourcePerQueuePercent = 
+        cs.getConfiguration().
+            getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
     int maxActiveApplications = 
         CSQueueUtils.computeMaxActiveApplications(
             cs.getClusterResources(), this.minimumAllocation,
-            maxAMResourcePercent, absoluteMaxCapacity);
+            maxAMResourcePerQueuePercent, absoluteMaxCapacity);
     this.maxActiveAppsUsingAbsCap = 
             CSQueueUtils.computeMaxActiveApplications(
                 cs.getClusterResources(), this.minimumAllocation,
-                maxAMResourcePercent, absoluteCapacity);
+                maxAMResourcePerQueuePercent, absoluteCapacity);
     int maxActiveApplicationsPerUser = 
         CSQueueUtils.computeMaxActiveApplicationsPerUser(maxActiveAppsUsingAbsCap, userLimit, 
             userLimitFactor);
@@ -265,15 +269,16 @@ public class LeafQueue implements CSQueue {
         "userLimitFactor = " + userLimitFactor +
         " [= configuredUserLimitFactor ]" + "\n" +
         "maxApplications = " + maxApplications +
-        " [= (int)(configuredMaximumSystemApplications * absoluteCapacity) ]" + 
+        " [= configuredMaximumSystemApplicationsPerQueue or" + 
+        " (int)(configuredMaximumSystemApplications * absoluteCapacity)]" + 
         "\n" +
         "maxApplicationsPerUser = " + maxApplicationsPerUser +
         " [= (int)(maxApplications * (userLimit / 100.0f) * " +
         "userLimitFactor) ]" + "\n" +
         "maxActiveApplications = " + maxActiveApplications +
         " [= max(" + 
-        "(int)ceil((clusterResourceMemory / minimumAllocation) *" + 
-        "maxAMResourcePercent * absoluteMaxCapacity)," + 
+        "(int)ceil((clusterResourceMemory / minimumAllocation) * " + 
+        "maxAMResourcePerQueuePercent * absoluteMaxCapacity)," + 
         "1) ]" + "\n" +
         "maxActiveAppsUsingAbsCap = " + maxActiveAppsUsingAbsCap +
         " [= max(" + 
@@ -290,7 +295,7 @@ public class LeafQueue implements CSQueue {
         "(clusterResourceMemory * absoluteCapacity)]" + "\n" +
         "absoluteUsedCapacity = " + absoluteUsedCapacity +
         " [= usedResourcesMemory / clusterResourceMemory]" + "\n" +
-        "maxAMResourcePercent = " + maxAMResourcePercent +
+        "maxAMResourcePerQueuePercent = " + maxAMResourcePerQueuePercent +
         " [= configuredMaximumAMResourcePercent ]" + "\n" +
         "minimumAllocationFactor = " + minimumAllocationFactor +
         " [= (float)(maximumAllocationMemory - minimumAllocationMemory) / " +
@@ -1387,11 +1392,11 @@ public class LeafQueue implements CSQueue {
     maxActiveApplications = 
         CSQueueUtils.computeMaxActiveApplications(
             clusterResource, minimumAllocation, 
-            maxAMResourcePercent, absoluteMaxCapacity);
+            maxAMResourcePerQueuePercent, absoluteMaxCapacity);
     maxActiveAppsUsingAbsCap = 
-            CSQueueUtils.computeMaxActiveApplications(
-                clusterResource, minimumAllocation, 
-                maxAMResourcePercent, absoluteCapacity);
+        CSQueueUtils.computeMaxActiveApplications(
+            clusterResource, minimumAllocation, 
+            maxAMResourcePerQueuePercent, absoluteCapacity);
     maxActiveApplicationsPerUser = 
         CSQueueUtils.computeMaxActiveApplicationsPerUser(
             maxActiveAppsUsingAbsCap, userLimit, userLimitFactor);

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java

@@ -158,7 +158,9 @@ public class TestApplicationLimits {
     int expectedMaxActiveApps = 
         Math.max(1, 
             (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * 
-                   csConf.getMaximumApplicationMasterResourcePercent() *
+                   csConf.
+                     getMaximumApplicationMasterResourcePerQueuePercent(
+                                                        queue.getQueuePath()) *
                    queue.getAbsoluteMaximumCapacity()));
     assertEquals(expectedMaxActiveApps, 
                  queue.getMaximumActiveApplications());
@@ -183,7 +185,9 @@ public class TestApplicationLimits {
     expectedMaxActiveApps = 
         Math.max(1, 
             (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * 
-                   csConf.getMaximumApplicationMasterResourcePercent() *
+                   csConf.
+                     getMaximumApplicationMasterResourcePerQueuePercent(
+                                                        queue.getQueuePath()) *
                    queue.getAbsoluteMaximumCapacity()));
     assertEquals(expectedMaxActiveApps, 
                  queue.getMaximumActiveApplications());
@@ -200,6 +204,72 @@ public class TestApplicationLimits {
         (int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()),
         queue.getMetrics().getAvailableMB()
         );
+
+    // should return -1 if per queue setting not set
+    assertEquals((int)csConf.UNDEFINED, csConf.getMaximumApplicationsPerQueue(queue.getQueuePath()));
+    int expectedMaxApps =  (int)(csConf.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS * 
+        queue.getAbsoluteCapacity());
+    assertEquals(expectedMaxApps, queue.getMaxApplications());
+
+    int expectedMaxAppsPerUser = (int)(expectedMaxApps *
+        (queue.getUserLimit()/100.0f) * queue.getUserLimitFactor());
+    assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser());
+
+    // should default to global setting if per queue setting not set
+    assertEquals((long) csConf.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, 
+        (long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(queue.getQueuePath()));
+
+    // Change the per-queue max AM resources percentage.
+    csConf.setFloat(
+      "yarn.scheduler.capacity." + 
+          queue.getQueuePath() + 
+          ".maximum-am-resource-percent",
+      0.5f);
+    // Re-create queues to get new configs.
+    queues = new HashMap<String, CSQueue>();
+    root = 
+        CapacityScheduler.parseQueue(csContext, csConf, null, "root", 
+            queues, queues, 
+            CapacityScheduler.queueComparator, 
+            CapacityScheduler.applicationComparator,
+            TestUtils.spyHook);
+    clusterResource = Resources.createResource(100 * 16 * GB);
+
+    queue = (LeafQueue)queues.get(A);
+    expectedMaxActiveApps = 
+        Math.max(1, 
+            (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * 
+                   csConf.
+                     getMaximumApplicationMasterResourcePerQueuePercent(
+                                                        queue.getQueuePath()) *
+                   queue.getAbsoluteMaximumCapacity()));
+
+    assertEquals((long) 0.5, 
+        (long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(queue.getQueuePath()));
+    assertEquals(expectedMaxActiveApps, 
+        queue.getMaximumActiveApplications());
+
+    // Change the per-queue max applications.
+    csConf.setInt(
+      "yarn.scheduler.capacity." + 
+          queue.getQueuePath() + 
+          ".maximum-applications", 9999);
+    // Re-create queues to get new configs.
+    queues = new HashMap<String, CSQueue>();
+    root = 
+        CapacityScheduler.parseQueue(csContext, csConf, null, "root", 
+            queues, queues, 
+            CapacityScheduler.queueComparator, 
+            CapacityScheduler.applicationComparator,
+            TestUtils.spyHook);
+
+    queue = (LeafQueue)queues.get(A);
+    assertEquals(9999, (int)csConf.getMaximumApplicationsPerQueue(queue.getQueuePath()));
+    assertEquals(9999, queue.getMaxApplications());
+
+    expectedMaxAppsPerUser = (int)(9999 *
+        (queue.getUserLimit()/100.0f) * queue.getUserLimitFactor());
+    assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser());
   }
   
   @Test

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm

@@ -240,17 +240,24 @@ Hadoop MapReduce Next Generation - Capacity Scheduler
 *--------------------------------------+--------------------------------------+
 || Property                            || Description                         |
 *--------------------------------------+--------------------------------------+
-| <<<yarn.scheduler.capacity.maximum-applications>>> | |
+| <<<yarn.scheduler.capacity.maximum-applications>>> /  |
+| <<<yarn.scheduler.capacity.<queue-path>.maximum-applications>>>  | |
 | | Maximum number of applications in the system which can be concurrently |
 | | active both running and pending. Limits on each queue are directly |
 | | proportional to their queue capacities and user limits. This is a 
 | | hard limit and any applications submitted when this limit is reached will |
-| | be rejected. Default is 10000.|
+| | be rejected. Default is 10000. This can be set for all queues with |
+| | <<<yarn.scheduler.capacity.maximum-applications>>> and can also be overridden on a  |
+| | per queue basis by setting <<<yarn.scheduler.capacity.<queue-path>.maximum-applications>>>. |
 *--------------------------------------+--------------------------------------+
-| yarn.scheduler.capacity.maximum-am-resource-percent | |
+| <<<yarn.scheduler.capacity.maximum-am-resource-percent>>> / |
+| <<<yarn.scheduler.capacity.<queue-path>.maximum-am-resource-percent>>> | |
 | | Maximum percent of resources in the cluster which can be used to run |
-| | application masters - controls number of concurrent running applications. |
-| | Specified as a float - ie 0.5 = 50%. Default is 10%. |
+| | application masters - controls number of concurrent active applications. Limits on each |
+| | queue are directly proportional to their queue capacities and user limits. |
+| | Specified as a float - ie 0.5 = 50%. Default is 10%. This can be set for all queues with |
+| | <<<yarn.scheduler.capacity.maximum-am-resource-percent>>> and can also be overridden on a  |
+| | per queue basis by setting <<<yarn.scheduler.capacity.<queue-path>.maximum-am-resource-percent>>> |
 *--------------------------------------+--------------------------------------+
 
     * Queue Administration & Permissions