YARN-9781. SchedConfCli to get current stored scheduler configuration. Contributed by Prabhu Joseph

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java

@@ -38,13 +38,24 @@ import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.dao.ConfInfo;
 import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
 import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
 
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.Marshaller;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response.Status;
+import java.io.StringReader;
+import java.io.StringWriter;
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.URL;
@@ -65,12 +76,16 @@ public class SchedConfCLI extends Configured implements Tool {
   private static final String REMOVE_QUEUES_OPTION = "removeQueues";
   private static final String UPDATE_QUEUES_OPTION = "updateQueues";
   private static final String GLOBAL_OPTIONS = "globalUpdates";
+  private static final String GET_SCHEDULER_CONF = "getConf";
   private static final String FORMAT_CONF = "formatConfig";
   private static final String HELP_CMD = "help";
 
   private static final String CONF_ERR_MSG = "Specify configuration key " +
       "value as confKey=confVal.";
 
+  private SSLFactory sslFactory;
+  private Client client;
+
   public SchedConfCLI() {
     super(new YarnConfiguration());
   }
@@ -93,6 +108,8 @@ public class SchedConfCLI extends Configured implements Tool {
         "Update queue configurations");
     opts.addOption("global", GLOBAL_OPTIONS, true,
         "Update global scheduler configurations");
+    opts.addOption("getconf", GET_SCHEDULER_CONF, false,
+        "Get current scheduler configurations");
     opts.addOption("format", FORMAT_CONF, false,
         "Format Scheduler Configuration and reload from" +
         " capacity-scheduler.xml");
@@ -115,6 +132,7 @@ public class SchedConfCLI extends Configured implements Tool {
 
     boolean hasOption = false;
     boolean format = false;
+    boolean getConf = false;
     SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
     try {
       if (parsedCli.hasOption(ADD_QUEUES_OPTION)) {
@@ -139,6 +157,10 @@ public class SchedConfCLI extends Configured implements Tool {
         hasOption = true;
         format = true;
       }
+      if (parsedCli.hasOption(GET_SCHEDULER_CONF)) {
+        hasOption = true;
+        getConf = true;
+      }
 
     } catch (IllegalArgumentException e) {
       System.err.println(e.getMessage());
@@ -154,27 +176,98 @@ public class SchedConfCLI extends Configured implements Tool {
     Configuration conf = getConf();
     if (format) {
       return WebAppUtils.execOnActiveRM(conf, this::formatSchedulerConf, null);
+    } else if (getConf) {
+      return WebAppUtils.execOnActiveRM(conf, this::getSchedulerConf, null);
     } else {
       return WebAppUtils.execOnActiveRM(conf,
           this::updateSchedulerConfOnRMNode, updateInfo);
     }
   }
 
-  @VisibleForTesting
-  int formatSchedulerConf(String webAppAddress, WebResource resource)
+  private static void prettyFormatWithIndent(String input, int indent)
       throws Exception {
+    Source xmlInput = new StreamSource(new StringReader(input));
+    StringWriter sw = new StringWriter();
+    StreamResult xmlOutput = new StreamResult(sw);
+    TransformerFactory transformerFactory = TransformerFactory.newInstance();
+    transformerFactory.setAttribute("indent-number", indent);
+    Transformer transformer = transformerFactory.newTransformer();
+    transformer.setOutputProperty(OutputKeys.INDENT, "yes");
+    transformer.transform(xmlInput, xmlOutput);
+    System.out.println(xmlOutput.getWriter().toString());
+  }
+
+  private WebResource initializeWebResource(String webAppAddress) {
     Configuration conf = getConf();
-    SSLFactory clientSslFactory = null;
     if (YarnConfiguration.useHttps(conf)) {
-      clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+      sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
     }
-    Client webServiceClient = createWebServiceClient(clientSslFactory);
+    client = createWebServiceClient(sslFactory);
+    return client.resource(webAppAddress);
+  }
+
+  private void destroyClient() {
+    if (client != null) {
+      client.destroy();
+    }
+    if (sslFactory != null) {
+      sslFactory.destroy();
+    }
+  }
+
+  @VisibleForTesting
+  int getSchedulerConf(String webAppAddress, WebResource resource)
+      throws Exception {
     ClientResponse response = null;
     resource = (resource != null) ? resource :
-        webServiceClient.resource(webAppAddress);
+        initializeWebResource(webAppAddress);
+    try {
+      Builder builder;
+      if (UserGroupInformation.isSecurityEnabled()) {
+        builder = resource
+            .path("ws").path("v1").path("cluster")
+            .path("scheduler-conf").accept(MediaType.APPLICATION_XML);
+      } else {
+        builder = resource
+            .path("ws").path("v1").path("cluster").path("scheduler-conf")
+            .queryParam("user.name", UserGroupInformation.getCurrentUser()
+            .getShortUserName()).accept(MediaType.APPLICATION_XML);
+      }
+      response = builder.get(ClientResponse.class);
+      if (response != null) {
+        if (response.getStatus() == Status.OK.getStatusCode()) {
+          ConfInfo schedulerConf = response.getEntity(ConfInfo.class);
+          JAXBContext jaxbContext = JAXBContext.newInstance(ConfInfo.class);
+          Marshaller jaxbMarshaller = jaxbContext.createMarshaller();
+          StringWriter sw = new StringWriter();
+          jaxbMarshaller.marshal(schedulerConf, sw);
+          prettyFormatWithIndent(sw.toString(), 2);
+          return 0;
+        } else {
+          System.err.println("Failed to get scheduler configuration: "
+              + response.getEntity(String.class));
+        }
+      } else {
+        System.err.println("Failed to get scheduler configuration: " +
+            "null response");
+      }
+      return -1;
+    } finally {
+      if (response != null) {
+        response.close();
+      }
+      destroyClient();
+    }
+  }
 
+  @VisibleForTesting
+  int formatSchedulerConf(String webAppAddress, WebResource resource)
+      throws Exception {
+    ClientResponse response = null;
+    resource = (resource != null) ? resource :
+        initializeWebResource(webAppAddress);
     try {
-      Builder builder = null;
+      Builder builder;
       if (UserGroupInformation.isSecurityEnabled()) {
         builder = resource
             .path("ws").path("v1").path("cluster")
@@ -206,27 +299,15 @@ public class SchedConfCLI extends Configured implements Tool {
       if (response != null) {
         response.close();
       }
-      if (webServiceClient != null) {
-        webServiceClient.destroy();
-      }
-      if (clientSslFactory != null) {
-        clientSslFactory.destroy();
-      }
+      destroyClient();
     }
   }
 
   @VisibleForTesting
   int updateSchedulerConfOnRMNode(String webAppAddress,
       SchedConfUpdateInfo updateInfo) throws Exception {
-    Configuration conf = getConf();
-    SSLFactory clientSslFactory = null;
-    if (YarnConfiguration.useHttps(conf)) {
-      clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
-    }
-    Client webServiceClient = createWebServiceClient(clientSslFactory);
     ClientResponse response = null;
-    WebResource resource = webServiceClient.resource(webAppAddress);
-
+    WebResource resource = initializeWebResource(webAppAddress);
     try {
       Builder builder = null;
       if (UserGroupInformation.isSecurityEnabled()) {
@@ -258,12 +339,7 @@ public class SchedConfCLI extends Configured implements Tool {
       if (response != null) {
         response.close();
       }
-      if (webServiceClient != null) {
-        webServiceClient.destroy();
-      }
-      if (clientSslFactory != null) {
-        clientSslFactory.destroy();
-      }
+      destroyClient();
     }
   }
 
@@ -380,7 +456,8 @@ public class SchedConfCLI extends Configured implements Tool {
         + "[-update \"queueUpdatePath1:confKey1=confVal1\"] "
         + "[-global globalConfKey1=globalConfVal1,"
         + "globalConfKey2=globalConfVal2] "
-        + "[-format]\n"
+        + "[-format] "
+        + "[-getconf]\n"
         + "Example (adding queues): yarn schedulerconf -add "
         + "\"root.a.a1:capacity=100,maximum-capacity=100;root.a.a2:capacity=0,"
         + "maximum-capacity=0\"\n"
@@ -393,6 +470,8 @@ public class SchedConfCLI extends Configured implements Tool {
         + "-global yarn.scheduler.capacity.maximum-applications=10000\n"
         + "Example (format scheduler configuration): yarn schedulerconf "
         + "-format\n"
+        + "Example (get scheduler configuration): yarn schedulerconf "
+        + "-getconf\n"
         + "Note: This is an alpha feature, the syntax/options are subject to "
         + "change, please run at your own risk.");
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java

@@ -200,6 +200,38 @@ public class TestSchedConfCLI extends JerseyTestBase {
     config.setMaximumCapacity(a, 100f);
   }
 
+  private void cleanUp() throws Exception {
+    if (rm != null) {
+      rm.stop();
+    }
+    CONF_FILE.delete();
+    if (CONF_FILE.exists()) {
+      throw new RuntimeException("Failed to delete configuration file");
+    }
+    if (OLD_CONF_FILE.exists()) {
+      if (!OLD_CONF_FILE.renameTo(CONF_FILE)) {
+        throw new RuntimeException("Failed to re-copy old" +
+            " configuration file");
+      }
+    }
+    super.tearDown();
+  }
+
+  @Test(timeout = 10000)
+  public void testGetSchedulerConf() throws Exception {
+    try {
+      super.setUp();
+      GuiceServletConfig.setInjector(
+          Guice.createInjector(new WebServletModule()));
+      int exitCode = cli.getSchedulerConf("", resource());
+      assertEquals("SchedConfCLI failed to run", 0, exitCode);
+      assertTrue("Failed to get scheduler configuration",
+          sysOutStream.toString().contains("testqueue"));
+    } finally {
+      cleanUp();
+    }
+  }
+
   @Test(timeout = 10000)
   public void testFormatSchedulerConf() throws Exception {
     try {
@@ -229,17 +261,7 @@ public class TestSchedConfCLI extends JerseyTestBase {
       schedulerConf = provider.getConfiguration();
       assertNull(schedulerConf.get("schedKey1"));
     } finally {
-      if (rm != null) {
-        rm.stop();
-      }
-      CONF_FILE.delete();
-      if (OLD_CONF_FILE.exists()) {
-        if (!OLD_CONF_FILE.renameTo(CONF_FILE)) {
-          throw new RuntimeException("Failed to re-copy old" +
-              " configuration file");
-        }
-      }
-      super.tearDown();
+      cleanUp();
     }
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfInfo.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/ConfInfo.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+package org.apache.hadoop.yarn.webapp.dao;
 
 import org.apache.hadoop.conf.Configuration;
 
@@ -24,11 +24,14 @@ import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlRootElement;
 import java.util.ArrayList;
 
+/**
+ * This class stores the Scheduler Configuration.
+ */
 @XmlRootElement(name = "configuration")
 @XmlAccessorType(XmlAccessType.FIELD)
 public class ConfInfo {
 
-  protected ArrayList<ConfItem> property = new ArrayList<>();
+  private ArrayList<ConfItem> property = new ArrayList<>();
 
   public ConfInfo() {
   } // JAXB needs this
@@ -46,6 +49,9 @@ public class ConfInfo {
     return property;
   }
 
+  /**
+   * This class stores the Configuration Property.
+   */
   @XmlAccessorType(XmlAccessType.FIELD)
   public static class ConfItem {
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java

@@ -197,7 +197,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfigVersionInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfInfo;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.server.webapp.WebServices;
@@ -210,6 +209,7 @@ import org.apache.hadoop.yarn.webapp.BadRequestException;
 import org.apache.hadoop.yarn.webapp.ForbiddenException;
 import org.apache.hadoop.yarn.webapp.NotFoundException;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.dao.ConfInfo;
 import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;