YARN-10782. Extend /scheduler endpoint with template properties. Contributed by Andras Gyori

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AutoQueueTemplatePropertiesInfo.java

@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LeafQueueTemplateInfo.ConfItem;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.ArrayList;
+
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class AutoQueueTemplatePropertiesInfo {
+
+  private ArrayList<ConfItem> property =
+      new ArrayList<>();
+
+  public AutoQueueTemplatePropertiesInfo() {
+  }
+
+  public ArrayList<ConfItem> getProperty() {
+    return property;
+  }
+
+  public void add(ConfItem confItem) {
+    property.add(confItem);
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java

@@ -62,6 +62,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
   protected String creationMethod;
   protected String autoCreationEligibility;
   protected String defaultNodeLabelExpression;
+  protected AutoQueueTemplatePropertiesInfo autoQueueTemplateProperties;
 
   @XmlTransient
   static final float EPSILON = 1e-8f;
@@ -109,6 +110,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
     if (parent instanceof ParentQueue) {
       orderingPolicyInfo = ((ParentQueue) parent).getQueueOrderingPolicy()
           .getConfigName();
+      autoQueueTemplateProperties = CapacitySchedulerInfoHelper
+            .getAutoCreatedTemplate((ParentQueue) parent);
     }
     mode = CapacitySchedulerInfoHelper.getMode(parent);
     queueType = CapacitySchedulerInfoHelper.getQueueType(parent);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java

@@ -19,10 +19,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
 
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
-import java.util.regex.Pattern;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
@@ -92,6 +90,8 @@ public class CapacitySchedulerQueueInfo {
   protected String creationMethod;
   protected String autoCreationEligibility;
   protected String defaultNodeLabelExpression;
+  protected AutoQueueTemplatePropertiesInfo autoQueueTemplateProperties =
+      new AutoQueueTemplatePropertiesInfo();
 
   CapacitySchedulerQueueInfo() {
   };
@@ -174,6 +174,8 @@ public class CapacitySchedulerQueueInfo {
     if (q instanceof ParentQueue) {
       orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy()
           .getConfigName();
+      autoQueueTemplateProperties = CapacitySchedulerInfoHelper
+            .getAutoCreatedTemplate((ParentQueue) q);
     }
 
     String configuredCapacity = conf.get(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java

@@ -23,6 +23,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AutoQueueTemplatePropertiesInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LeafQueueTemplateInfo.ConfItem;
+
+import java.util.Map;
 
 /**
  * Helper class to describe a queue's type, its creation method and its
@@ -106,4 +110,17 @@ public class CapacitySchedulerInfoHelper {
       return AUTO_CREATION_OFF;
     }
   }
+
+  public static AutoQueueTemplatePropertiesInfo getAutoCreatedTemplate(
+      ParentQueue parent) {
+    AutoQueueTemplatePropertiesInfo propertiesInfo =
+        new AutoQueueTemplatePropertiesInfo();
+    for (Map.Entry<String, String> e :
+        parent.getAutoCreatedQueueTemplate().getTemplateProperties()
+            .entrySet()) {
+      propertiesInfo.add(new ConfItem(e.getKey(), e.getValue()));
+    }
+
+    return propertiesInfo;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java

@@ -369,7 +369,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
     JSONObject info = json.getJSONObject("scheduler");
     assertEquals("incorrect number of elements in: " + info, 1, info.length());
     info = info.getJSONObject("schedulerInfo");
-    assertEquals("incorrect number of elements in: " + info, 19, info.length());
+    assertEquals("incorrect number of elements in: " + info, 20, info.length());
     verifyClusterSchedulerGeneric(info.getString("type"),
         (float) info.getDouble("usedCapacity"),
         (float) info.getDouble("capacity"),
@@ -424,10 +424,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
   private void verifySubQueue(JSONObject info, String q,
       float parentAbsCapacity, float parentAbsMaxCapacity)
       throws JSONException, Exception {
-    int numExpectedElements = 34;
+    int numExpectedElements = 35;
     boolean isParentQueue = true;
     if (!info.has("queues")) {
-      numExpectedElements = 52;
+      numExpectedElements = 53;
       isParentQueue = false;
     }
     assertEquals("incorrect number of elements", numExpectedElements, info.length());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java

@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
 import static org.junit.Assert.assertEquals;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -40,6 +41,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
@@ -93,6 +95,7 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
     private String queueType;
     private String creationMethod;
     private String autoCreationEligibility;
+    private List<String[]> autoQueueTemplateProperties;
 
     public ExpectedQueueWithProperties(String path, float weight,
         float normalizedWeight, String queueType, String creationMethod,
@@ -103,6 +106,20 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
       this.queueType = queueType;
       this.creationMethod = creationMethod;
       this.autoCreationEligibility = autoCreationEligibility;
+      this.autoQueueTemplateProperties = new ArrayList<>();
+    }
+
+    ExpectedQueueWithProperties(
+        String path, float weight, float normalizedWeight, String queueType,
+        String creationMethod, String autoCreationEligibility,
+        List<String[]> autoQueueTemplateProperties) {
+      this.path = path;
+      this.weight = weight;
+      this.normalizedWeight = normalizedWeight;
+      this.queueType = queueType;
+      this.creationMethod = creationMethod;
+      this.autoCreationEligibility = autoCreationEligibility;
+      this.autoQueueTemplateProperties = autoQueueTemplateProperties;
     }
   }
 
@@ -260,6 +277,10 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
         .createWeightConfigWithAutoQueueCreationEnabled();
     config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
         YarnConfiguration.MEMORY_CONFIGURATION_STORE);
+    config.setInt(CapacitySchedulerConfiguration
+        .getQueuePrefix("root.autoParent1") +
+        AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX +
+        "maximum-applications", 300);
 
     initResourceManager(config);
     initAutoQueueHandler();
@@ -290,6 +311,9 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
         new ExpectedQueueWithProperties("root",
             EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE,
             PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF);
+    List<String[]> templateProperties = new ArrayList<>();
+    templateProperties.add(new String[] {"maximum-applications", "300"});
+
     validateSchedulerInfo(json, "weight",
         expectedRootQ,
         new ExpectedQueueWithProperties("root.auto1",
@@ -307,7 +331,8 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
         new ExpectedQueueWithProperties("root.autoParent1",
             EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
             EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights,
-            PARENT_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_FLEXIBLE),
+            PARENT_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_FLEXIBLE,
+            templateProperties),
         new ExpectedQueueWithProperties("root.default", 10.0f,
             10.0f / sumOfWeights,
             LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
@@ -474,6 +499,22 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
               queuePath,
           expectedQueue.creationMethod, obj.getString("creationMethod"));
 
+      if (!expectedQueue.autoQueueTemplateProperties.isEmpty()) {
+        JSONArray templates = obj.getJSONObject("autoQueueTemplateProperties")
+            .getJSONArray("property");
+        for (int j = 0; j < templates.length(); j++) {
+          JSONObject prop = templates.getJSONObject(j);
+          Assert.assertEquals("Auto creation eligible queue " +
+                  "template key do not match for queue" + queuePath,
+              expectedQueue.autoQueueTemplateProperties.get(j)[0],
+              prop.getString("name"));
+          Assert.assertEquals("Auto creation eligible queue " +
+                  "template value do not match for queue" + queuePath,
+              expectedQueue.autoQueueTemplateProperties.get(j)[1],
+              prop.getString("value"));
+        }
+      }
+
       Assert.assertEquals("Queue auto creation eligibility does not " +
               "match for queue " + queuePath,
           expectedQueue.autoCreationEligibility,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java

@@ -574,7 +574,7 @@ public class TestRMWebServicesForCSWithPartitions extends JerseyTestBase {
     JSONObject info = json.getJSONObject("scheduler");
     assertEquals("incorrect number of elements", 1, info.length());
     info = info.getJSONObject("schedulerInfo");
-    assertEquals("incorrect number of elements", 19, info.length());
+    assertEquals("incorrect number of elements", 20, info.length());
     JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES);
     JSONArray partitionsCapsArray =
         capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);