YARN-7451. Add missing tests to verify the presence of custom resources of RM apps and scheduler webservice endpoints (snemeth via rkanter)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java

@@ -479,7 +479,7 @@ public class AppInfo {
   public int getNumNonAMContainersPreempted() {
     return numNonAMContainerPreempted;
   }
-  
+
   public int getNumAMContainersPreempted() {
     return numAMContainerPreempted;
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java

@@ -41,8 +41,9 @@ public class SchedulerInfo {
   protected EnumSet<SchedulerResourceTypes> schedulingResourceTypes;
   protected int maximumClusterPriority;
 
+  // JAXB needs this
   public SchedulerInfo() {
-  } // JAXB needs this
+  }
 
   public SchedulerInfo(final ResourceManager rm) {
     ResourceScheduler rs = rm.getResourceScheduler();
@@ -74,7 +75,10 @@ public class SchedulerInfo {
   }
 
   public String getSchedulerResourceTypes() {
-    return Arrays.toString(minAllocResource.getResource().getResources());
+    if (minAllocResource != null) {
+      return Arrays.toString(minAllocResource.getResource().getResources());
+    }
+    return null;
   }
 
   public int getMaxClusterLevelAppPriority() {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java

@@ -48,6 +48,9 @@ import org.apache.log4j.spi.LoggingEvent;
 import org.junit.Assert;
 import org.junit.Test;
 
+/**
+ * Tests fair scheduler configuration.
+ */
 public class TestFairSchedulerConfiguration {
 
   private static final String A_CUSTOM_RESOURCE = "a-custom-resource";
@@ -242,12 +245,12 @@ public class TestFairSchedulerConfiguration {
         parseResourceConfigValue(" vcores = 75 % , memory-mb = 40 % , "
             + "test1 = 50 % ").getResource(clusterResource));
   }
-  
+
   @Test(expected = AllocationConfigurationException.class)
   public void testNoUnits() throws Exception {
     parseResourceConfigValue("1024");
   }
-  
+
   @Test(expected = AllocationConfigurationException.class)
   public void testOnlyMemory() throws Exception {
     parseResourceConfigValue("1024mb");
@@ -257,7 +260,7 @@ public class TestFairSchedulerConfiguration {
   public void testOnlyCPU() throws Exception {
     parseResourceConfigValue("1024vcores");
   }
-  
+
   @Test(expected = AllocationConfigurationException.class)
   public void testGibberish() throws Exception {
     parseResourceConfigValue("1o24vc0res");

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java

@@ -53,11 +53,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.api.records.QueueState;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
-import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
-import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
-import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.*;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -76,11 +72,12 @@ import org.apache.hadoop.yarn.webapp.JerseyTestBase;
 import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
-import org.eclipse.jetty.server.Response;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 import org.w3c.dom.NodeList;
@@ -96,6 +93,8 @@ import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
 import com.sun.jersey.test.framework.WebAppDescriptor;
 
 public class TestRMWebServices extends JerseyTestBase {
+  private static final Logger LOG =
+          LoggerFactory.getLogger(TestRMWebServices.class);
 
   private static MockRM rm;
 
@@ -472,19 +471,19 @@ public class TestRMWebServices extends JerseyTestBase {
     QueueMetrics metrics = rs.getRootQueueMetrics();
     ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
 
-    long totalMBExpect = 
+    long totalMBExpect =
         metrics.getAvailableMB() + metrics.getAllocatedMB();
-    long totalVirtualCoresExpect = 
+    long totalVirtualCoresExpect =
         metrics.getAvailableVirtualCores() + metrics.getAllocatedVirtualCores();
-    assertEquals("appsSubmitted doesn't match", 
+    assertEquals("appsSubmitted doesn't match",
         metrics.getAppsSubmitted(), submittedApps);
-    assertEquals("appsCompleted doesn't match", 
+    assertEquals("appsCompleted doesn't match",
         metrics.getAppsCompleted(), completedApps);
     assertEquals("reservedMB doesn't match",
         metrics.getReservedMB(), reservedMB);
-    assertEquals("availableMB doesn't match", 
+    assertEquals("availableMB doesn't match",
         metrics.getAvailableMB(), availableMB);
-    assertEquals("allocatedMB doesn't match", 
+    assertEquals("allocatedMB doesn't match",
         metrics.getAllocatedMB(), allocMB);
     assertEquals("reservedVirtualCores doesn't match",
         metrics.getReservedVirtualCores(), reservedVirtualCores);
@@ -597,11 +596,13 @@ public class TestRMWebServices extends JerseyTestBase {
 
   public void verifyClusterSchedulerFifo(JSONObject json) throws JSONException,
       Exception {
-    assertEquals("incorrect number of elements", 1, json.length());
+    assertEquals("incorrect number of elements in: " + json, 1, json.length());
     JSONObject info = json.getJSONObject("scheduler");
-    assertEquals("incorrect number of elements", 1, info.length());
+    assertEquals("incorrect number of elements in: " + info, 1, info.length());
     info = info.getJSONObject("schedulerInfo");
-    assertEquals("incorrect number of elements", 11, info.length());
+
+    LOG.debug("schedulerInfo: {}", info);
+    assertEquals("incorrect number of elements in: " + info, 11, info.length());
 
     verifyClusterSchedulerFifoGeneric(info.getString("type"),
         info.getString("qstate"), (float) info.getDouble("capacity"),

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java

@@ -79,7 +79,7 @@ import com.sun.jersey.test.framework.WebAppDescriptor;
 public class TestRMWebServicesApps extends JerseyTestBase {
 
   private static MockRM rm;
-  
+
   private static final int CONTAINER_MB = 1024;
 
   private static class WebServletModule extends ServletModule {
@@ -324,7 +324,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
     assertEquals("incorrect number of elements", 1, apps.length());
     array = apps.getJSONArray("app");
     assertEquals("incorrect number of elements", 2, array.length());
-    assertTrue("both app states of ACCEPTED and KILLED are not present", 
+    assertTrue("both app states of ACCEPTED and KILLED are not present",
         (array.getJSONObject(0).getString("state").equals("ACCEPTED") &&
         array.getJSONObject(1).getString("state").equals("KILLED")) ||
         (array.getJSONObject(0).getString("state").equals("KILLED") &&
@@ -375,12 +375,12 @@ public class TestRMWebServicesApps extends JerseyTestBase {
     assertEquals("incorrect number of elements", 1, apps.length());
     array = apps.getJSONArray("app");
     assertEquals("incorrect number of elements", 2, array.length());
-    assertTrue("both app states of ACCEPTED and KILLED are not present", 
+    assertTrue("both app states of ACCEPTED and KILLED are not present",
         (array.getJSONObject(0).getString("state").equals("ACCEPTED") &&
         array.getJSONObject(1).getString("state").equals("KILLED")) ||
         (array.getJSONObject(0).getString("state").equals("KILLED") &&
         array.getJSONObject(1).getString("state").equals("ACCEPTED")));
-    
+
     rm.stop();
   }
 
@@ -511,7 +511,8 @@ public class TestRMWebServicesApps extends JerseyTestBase {
     WebResource r = resource();
 
     ClientResponse response = r.path("ws").path("v1").path("cluster")
-        .path("apps").queryParam("finalStatus", FinalApplicationStatus.UNDEFINED.toString())
+        .path("apps").queryParam("finalStatus",
+                    FinalApplicationStatus.UNDEFINED.toString())
         .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
@@ -1804,7 +1805,8 @@ public class TestRMWebServicesApps extends JerseyTestBase {
     int numAttempt = 1;
     while (true) {
       // fail the AM by sending CONTAINER_FINISHED event without registering.
-      amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
+      amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1,
+              ContainerState.COMPLETE);
       rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FAILED);
       if (numAttempt == maxAppAttempts) {
         rm.waitForState(app1.getApplicationId(), RMAppState.FAILED);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java

@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.CustomResourceTypesConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoJsonVerifications;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoXmlVerifications;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.BufferedClientResponse;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.JsonCustomResourceTypeTestcase;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsJsonVerifications;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsXmlVerifications;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Before;
+import org.junit.Test;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import javax.ws.rs.core.MediaType;
+import java.util.ArrayList;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * This test verifies that custom resource types are correctly serialized to XML
+ * and JSON when HTTP GET request is sent to the resource: ws/v1/cluster/apps.
+ */
+public class TestRMWebServicesAppsCustomResourceTypes extends JerseyTestBase {
+
+  private static MockRM rm;
+  private static final int CONTAINER_MB = 1024;
+
+  private static class WebServletModule extends ServletModule {
+    @Override
+    protected void configureServlets() {
+      bind(JAXBContextResolver.class);
+      bind(RMWebServices.class);
+      bind(GenericExceptionHandler.class);
+      Configuration conf = new Configuration();
+      conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+          YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
+      conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
+          ResourceScheduler.class);
+      initResourceTypes(conf);
+      rm = new MockRM(conf);
+      bind(ResourceManager.class).toInstance(rm);
+      serve("/*").with(GuiceContainer.class);
+    }
+
+    private void initResourceTypes(Configuration conf) {
+      conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+          CustomResourceTypesConfigurationProvider.class.getName());
+      ResourceUtils.resetResourceTypes(conf);
+    }
+  }
+
+  @Before
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    createInjectorForWebServletModule();
+  }
+
+  private void createInjectorForWebServletModule() {
+    GuiceServletConfig
+        .setInjector(Guice.createInjector(new WebServletModule()));
+  }
+
+  public TestRMWebServicesAppsCustomResourceTypes() {
+    super(new WebAppDescriptor.Builder(
+        "org.apache.hadoop.yarn.server.resourcemanager.webapp")
+            .contextListenerClass(GuiceServletConfig.class)
+            .filterClass(com.google.inject.servlet.GuiceFilter.class)
+            .contextPath("jersey-guice-filter").servletPath("/").build());
+  }
+
+  @Test
+  public void testRunningAppXml() throws Exception {
+    rm.start();
+    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+    RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
+    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
+    am1.allocate("*", 2048, 1, new ArrayList<>());
+    amNodeManager.nodeHeartbeat(true);
+
+    WebResource r = resource();
+    WebResource path = r.path("ws").path("v1").path("cluster").path("apps");
+    ClientResponse response =
+        path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+
+    XmlCustomResourceTypeTestCase testCase =
+            new XmlCustomResourceTypeTestCase(path,
+                    new BufferedClientResponse(response));
+    testCase.verify(document -> {
+      NodeList apps = document.getElementsByTagName("apps");
+      assertEquals("incorrect number of apps elements", 1, apps.getLength());
+
+      NodeList appArray = ((Element)(apps.item(0)))
+              .getElementsByTagName("app");
+      assertEquals("incorrect number of app elements", 1, appArray.getLength());
+
+      verifyAppsXML(appArray, app1);
+    });
+
+    rm.stop();
+  }
+
+  @Test
+  public void testRunningAppJson() throws Exception {
+    rm.start();
+    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+    RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
+    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
+    am1.allocate("*", 2048, 1, new ArrayList<>());
+    amNodeManager.nodeHeartbeat(true);
+
+    WebResource r = resource();
+    WebResource path = r.path("ws").path("v1").path("cluster").path("apps");
+    ClientResponse response =
+        path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+
+    JsonCustomResourceTypeTestcase testCase =
+        new JsonCustomResourceTypeTestcase(path,
+            new BufferedClientResponse(response));
+    testCase.verify(json -> {
+      try {
+        assertEquals("incorrect number of apps elements", 1, json.length());
+        JSONObject apps = json.getJSONObject("apps");
+        assertEquals("incorrect number of app elements", 1, apps.length());
+        JSONArray array = apps.getJSONArray("app");
+        assertEquals("incorrect count of app", 1, array.length());
+
+        verifyAppInfoJson(array.getJSONObject(0), app1);
+      } catch (JSONException e) {
+        throw new RuntimeException(e);
+      }
+    });
+
+    rm.stop();
+  }
+
+  private void verifyAppsXML(NodeList appArray, RMApp app) {
+    for (int i = 0; i < appArray.getLength(); i++) {
+      Element element = (Element) appArray.item(i);
+      AppInfoXmlVerifications.verify(element, app);
+
+      NodeList resourceRequests =
+          element.getElementsByTagName("resourceRequests");
+      assertEquals(1, resourceRequests.getLength());
+      Node resourceRequest = resourceRequests.item(0);
+      ResourceRequest rr =
+          ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
+              .getApplicationAttempt(
+                  app.getCurrentAppAttempt().getAppAttemptId())
+              .getAppSchedulingInfo().getAllResourceRequests().get(0);
+      ResourceRequestsXmlVerifications.verifyWithCustomResourceTypes(
+              (Element) resourceRequest, rr,
+          CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
+    }
+  }
+
+  private void verifyAppInfoJson(JSONObject info, RMApp app) throws
+          JSONException {
+    int expectedNumberOfElements = getExpectedNumberOfElements(app);
+
+    assertEquals("incorrect number of elements", expectedNumberOfElements,
+        info.length());
+
+    AppInfoJsonVerifications.verify(info, app);
+
+    JSONArray resourceRequests = info.getJSONArray("resourceRequests");
+    JSONObject requestInfo = resourceRequests.getJSONObject(0);
+    ResourceRequest rr =
+        ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
+            .getApplicationAttempt(app.getCurrentAppAttempt().getAppAttemptId())
+            .getAppSchedulingInfo().getAllResourceRequests().get(0);
+
+    ResourceRequestsJsonVerifications.verifyWithCustomResourceTypes(
+            requestInfo, rr,
+            CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
+  }
+
+  private int getExpectedNumberOfElements(RMApp app) {
+    int expectedNumberOfElements = 40 + 2; // 2 -> resourceRequests
+    if (app.getApplicationSubmissionContext()
+        .getNodeLabelExpression() != null) {
+      expectedNumberOfElements++;
+    }
+
+    if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) {
+      expectedNumberOfElements++;
+    }
+
+    if (AppInfo
+        .getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()) != null) {
+      expectedNumberOfElements++;
+    }
+    return expectedNumberOfElements;
+  }
+
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java

@@ -146,7 +146,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
     config.setUserLimitFactor(B2, 100.0f);
     config.setCapacity(B3, 0.5f);
     config.setUserLimitFactor(B3, 100.0f);
-    
+
     config.setQueues(A1, new String[] {"a1a", "a1b"});
     final String A1A = A1 + ".a1a";
     config.setCapacity(A1A, 85);
@@ -254,7 +254,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
     }
   }
 
-  public void verifySubQueueXML(Element qElem, String q, 
+  public void verifySubQueueXML(Element qElem, String q,
       float parentAbsCapacity, float parentAbsMaxCapacity)
       throws Exception {
     NodeList children = qElem.getChildNodes();
@@ -317,30 +317,34 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
 
   private void verifyClusterScheduler(JSONObject json) throws JSONException,
       Exception {
-    assertEquals("incorrect number of elements", 1, json.length());
+    assertEquals("incorrect number of elements in: " + json, 1, json.length());
     JSONObject info = json.getJSONObject("scheduler");
-    assertEquals("incorrect number of elements", 1, info.length());
+    assertEquals("incorrect number of elements in: " + info, 1, info.length());
     info = info.getJSONObject("schedulerInfo");
-    assertEquals("incorrect number of elements", 8, info.length());
+    assertEquals("incorrect number of elements in: " + info, 8, info.length());
     verifyClusterSchedulerGeneric(info.getString("type"),
         (float) info.getDouble("usedCapacity"),
         (float) info.getDouble("capacity"),
         (float) info.getDouble("maxCapacity"), info.getString("queueName"));
     JSONObject health = info.getJSONObject("health");
     assertNotNull(health);
-    assertEquals("incorrect number of elements", 3, health.length());
+    assertEquals("incorrect number of elements in: " + health, 3,
+        health.length());
     JSONArray operationsInfo = health.getJSONArray("operationsInfo");
-    assertEquals("incorrect number of elements", 4, operationsInfo.length());
+    assertEquals("incorrect number of elements in: " + health, 4,
+        operationsInfo.length());
     JSONArray lastRunDetails = health.getJSONArray("lastRunDetails");
-    assertEquals("incorrect number of elements", 3, lastRunDetails.length());
+    assertEquals("incorrect number of elements in: " + health, 3,
+        lastRunDetails.length());
 
     JSONArray arr = info.getJSONObject("queues").getJSONArray("queue");
-    assertEquals("incorrect number of elements", 2, arr.length());
+    assertEquals("incorrect number of elements in: " + arr, 2, arr.length());
 
     // test subqueues
     for (int i = 0; i < arr.length(); i++) {
       JSONObject obj = arr.getJSONObject(i);
-      String q = CapacitySchedulerConfiguration.ROOT + "." + obj.getString("queueName");
+      String q = CapacitySchedulerConfiguration.ROOT + "." +
+              obj.getString("queueName");
       verifySubQueue(obj, q, 100, 100);
     }
   }
@@ -355,7 +359,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
     assertTrue("queueName doesn't match", "root".matches(queueName));
   }
 
-  private void verifySubQueue(JSONObject info, String q, 
+  private void verifySubQueue(JSONObject info, String q,
       float parentAbsCapacity, float parentAbsMaxCapacity)
       throws JSONException, Exception {
     int numExpectedElements = 20;
@@ -464,7 +468,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
         csConf.getUserLimitFactor(q), info.userLimitFactor, 1e-3f);
   }
 
-  //Return a child Node of node with the tagname or null if none exists 
+  //Return a child Node of node with the tagname or null if none exists
   private Node getChildNodeByName(Node node, String tagname) {
     NodeList nodeList = node.getChildNodes();
     for (int i=0; i < nodeList.getLength(); ++i) {
@@ -514,7 +518,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
           for (int j=0; j<users.getLength(); ++j) {
             Node user = users.item(j);
             String username = getChildNodeByName(user, "username")
-              .getTextContent(); 
+                .getTextContent();
             assertTrue(username.equals("user1") || username.equals("user2"));
             //Should be a parsable integer
             Integer.parseInt(getChildNodeByName(getChildNodeByName(user,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java

@@ -42,6 +42,8 @@ import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response.Status;
@@ -59,6 +61,8 @@ import static org.junit.Assert.assertNull;
  * Test scheduler configuration mutation via REST API.
  */
 public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
+  private static final Logger LOG = LoggerFactory
+          .getLogger(TestRMWebServicesConfigurationMutation.class);
 
   private static final File CONF_FILE = new File(new File("target",
       "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE);
@@ -396,6 +400,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
             .entity(YarnWebServiceUtils.toJson(updateInfo,
                 SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
             .put(ClientResponse.class);
+    LOG.debug("Response headers: " + response.getHeaders());
     assertEquals(Status.OK.getStatusCode(), response.getStatus());
     CapacitySchedulerConfiguration newCSConf = cs.getConfiguration();
     assertEquals(0.2f, newCSConf

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java

@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,13 +16,14 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.yarn.server.resourcemanager.webapp;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import javax.ws.rs.core.MediaType;
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
 
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
 import org.apache.hadoop.http.JettyUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
@@ -30,6 +31,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
+
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
 import org.apache.hadoop.yarn.webapp.JerseyTestBase;
@@ -38,18 +42,18 @@ import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 import org.junit.Before;
 import org.junit.Test;
+import javax.ws.rs.core.MediaType;
 
-import com.google.inject.Guice;
-import com.google.inject.servlet.ServletModule;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.WebAppDescriptor;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
 
+/**
+ * Tests RM Webservices fair scheduler resources.
+ */
 public class TestRMWebServicesFairScheduler extends JerseyTestBase {
   private static MockRM rm;
   private static YarnConfiguration conf;
-  
+
   private static class WebServletModule extends ServletModule {
     @Override
     protected void configureServlets() {
@@ -58,7 +62,7 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
       bind(GenericExceptionHandler.class);
       conf = new YarnConfiguration();
       conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
-        ResourceScheduler.class);
+          ResourceScheduler.class);
       rm = new MockRM(conf);
       bind(ResourceManager.class).toInstance(rm);
       serve("/*").with(GuiceContainer.class);
@@ -66,32 +70,32 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
   }
 
   static {
-    GuiceServletConfig.setInjector(
-        Guice.createInjector(new WebServletModule()));
+    GuiceServletConfig
+        .setInjector(Guice.createInjector(new WebServletModule()));
   }
 
   @Before
   @Override
   public void setUp() throws Exception {
     super.setUp();
-    GuiceServletConfig.setInjector(
-        Guice.createInjector(new WebServletModule()));
+    GuiceServletConfig
+        .setInjector(Guice.createInjector(new WebServletModule()));
   }
 
   public TestRMWebServicesFairScheduler() {
     super(new WebAppDescriptor.Builder(
         "org.apache.hadoop.yarn.server.resourcemanager.webapp")
-        .contextListenerClass(GuiceServletConfig.class)
-        .filterClass(com.google.inject.servlet.GuiceFilter.class)
-        .contextPath("jersey-guice-filter").servletPath("/").build());
+            .contextListenerClass(GuiceServletConfig.class)
+            .filterClass(com.google.inject.servlet.GuiceFilter.class)
+            .contextPath("jersey-guice-filter").servletPath("/").build());
   }
-  
+
   @Test
-  public void testClusterScheduler() throws JSONException, Exception {
+  public void testClusterScheduler() throws JSONException {
     WebResource r = resource();
-    ClientResponse response = r.path("ws").path("v1").path("cluster")
-        .path("scheduler").accept(MediaType.APPLICATION_JSON)
-        .get(ClientResponse.class);
+    ClientResponse response =
+        r.path("ws").path("v1").path("cluster").path("scheduler")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     JSONObject json = response.getEntity(JSONObject.class);
@@ -99,52 +103,51 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
   }
 
   @Test
-  public void testClusterSchedulerSlash() throws JSONException, Exception {
+  public void testClusterSchedulerSlash() throws JSONException {
     WebResource r = resource();
-    ClientResponse response = r.path("ws").path("v1").path("cluster")
-        .path("scheduler/").accept(MediaType.APPLICATION_JSON)
-        .get(ClientResponse.class);
+    ClientResponse response =
+        r.path("ws").path("v1").path("cluster").path("scheduler/")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     JSONObject json = response.getEntity(JSONObject.class);
     verifyClusterScheduler(json);
   }
-  
+
   @Test
-  public void testClusterSchedulerWithSubQueues() throws JSONException,
-      Exception {
-    FairScheduler scheduler = (FairScheduler)rm.getResourceScheduler();
+  public void testClusterSchedulerWithSubQueues()
+      throws JSONException {
+    FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
     QueueManager queueManager = scheduler.getQueueManager();
     // create LeafQueue
     queueManager.getLeafQueue("root.q.subqueue1", true);
     queueManager.getLeafQueue("root.q.subqueue2", true);
 
     WebResource r = resource();
-    ClientResponse response = r.path("ws").path("v1").path("cluster")
-        .path("scheduler").accept(MediaType.APPLICATION_JSON)
-        .get(ClientResponse.class);
+    ClientResponse response =
+        r.path("ws").path("v1").path("cluster").path("scheduler")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     JSONObject json = response.getEntity(JSONObject.class);
     JSONArray subQueueInfo = json.getJSONObject("scheduler")
         .getJSONObject("schedulerInfo").getJSONObject("rootQueue")
-        .getJSONObject("childQueues").getJSONArray("queue")
-        .getJSONObject(1).getJSONObject("childQueues").getJSONArray("queue");
+        .getJSONObject("childQueues").getJSONArray("queue").getJSONObject(1)
+        .getJSONObject("childQueues").getJSONArray("queue");
     // subQueueInfo is consist of subqueue1 and subqueue2 info
     assertEquals(2, subQueueInfo.length());
 
     // Verify 'childQueues' field is omitted from FairSchedulerLeafQueueInfo.
     try {
       subQueueInfo.getJSONObject(1).getJSONObject("childQueues");
-      fail("FairSchedulerQueueInfo should omit field 'childQueues'" +
-           "if child queue is empty.");
+      fail("FairSchedulerQueueInfo should omit field 'childQueues'"
+          + "if child queue is empty.");
     } catch (JSONException je) {
       assertEquals("JSONObject[\"childQueues\"] not found.", je.getMessage());
     }
   }
 
-  private void verifyClusterScheduler(JSONObject json) throws JSONException,
-      Exception {
+  private void verifyClusterScheduler(JSONObject json) throws JSONException {
     assertEquals("incorrect number of elements", 1, json.length());
     JSONObject info = json.getJSONObject("scheduler");
     assertEquals("incorrect number of elements", 1, info.length());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java

@@ -457,7 +457,7 @@ public class TestRMWebServicesSchedulerActivities
       if (object.getClass() == JSONObject.class) {
         assertEquals("Number of allocations is wrong", 1, realValue);
       } else if (object.getClass() == JSONArray.class) {
-        assertEquals("Number of allocations is wrong",
+        assertEquals("Number of allocations is wrong in: " + object,
             ((JSONArray) object).length(), realValue);
       }
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java

@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.LocalConfigurationProvider;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static java.util.stream.Collectors.toList;
+
+/**
+ * This class can generate an XML configuration file of custom resource types.
+ * See createInitialResourceTypes for the default values. All custom resource
+ * type is prefixed with CUSTOM_RESOURCE_PREFIX. Please use the
+ * getConfigurationInputStream method to get an InputStream of the XML. If you
+ * want to have different number of resources in your tests, please see usages
+ * of this class in this test class:
+ * {@link TestRMWebServicesFairSchedulerCustomResourceTypes}
+ *
+ */
+public class CustomResourceTypesConfigurationProvider
+    extends LocalConfigurationProvider {
+
+  private static class CustomResourceTypes {
+    private int count;
+    private String xml;
+
+    CustomResourceTypes(String xml, int count) {
+      this.xml = xml;
+      this.count = count;
+    }
+
+    public int getCount() {
+      return count;
+    }
+
+    public String getXml() {
+      return xml;
+    }
+  }
+
+  private static final String CUSTOM_RESOURCE_PREFIX = "customResource-";
+
+  private static CustomResourceTypes customResourceTypes =
+      createInitialResourceTypes();
+
+  private static CustomResourceTypes createInitialResourceTypes() {
+    return createCustomResourceTypes(2);
+  }
+
+  private static CustomResourceTypes createCustomResourceTypes(int count) {
+    List<String> resourceTypeNames = generateResourceTypeNames(count);
+
+    List<String> resourceUnitXmlElements = IntStream.range(0, count)
+            .boxed()
+            .map(i -> getResourceUnitsXml(resourceTypeNames.get(i)))
+            .collect(toList());
+
+    StringBuilder sb = new StringBuilder("<configuration>\n");
+    sb.append(getResourceTypesXml(resourceTypeNames));
+
+    for (String resourceUnitXml : resourceUnitXmlElements) {
+      sb.append(resourceUnitXml);
+
+    }
+    sb.append("</configuration>");
+
+    return new CustomResourceTypes(sb.toString(), count);
+  }
+
+  private static List<String> generateResourceTypeNames(int count) {
+    return IntStream.range(0, count)
+            .boxed()
+            .map(i -> CUSTOM_RESOURCE_PREFIX + i)
+            .collect(toList());
+  }
+
+  private static String getResourceUnitsXml(String resource) {
+    return "<property>\n" + "<name>yarn.resource-types." + resource
+        + ".units</name>\n" + "<value>k</value>\n" + "</property>\n";
+  }
+
+  private static String getResourceTypesXml(List<String> resources) {
+    final String resourceTypes = makeCommaSeparatedString(resources);
+
+    return "<property>\n" + "<name>yarn.resource-types</name>\n" + "<value>"
+        + resourceTypes + "</value>\n" + "</property>\n";
+  }
+
+  private static String makeCommaSeparatedString(List<String> resources) {
+    return resources.stream().collect(Collectors.joining(","));
+  }
+
+  @Override
+  public InputStream getConfigurationInputStream(Configuration bootstrapConf,
+      String name) throws YarnException, IOException {
+    if (YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE.equals(name)) {
+      return new ByteArrayInputStream(
+          customResourceTypes.getXml().getBytes());
+    } else {
+      return super.getConfigurationInputStream(bootstrapConf, name);
+    }
+  }
+
+  public static void reset() {
+    customResourceTypes = createInitialResourceTypes();
+  }
+
+  public static void setNumberOfResourceTypes(int count) {
+    customResourceTypes = createCustomResourceTypes(count);
+  }
+
+  public static List<String> getCustomResourceTypes() {
+    return generateResourceTypeNames(customResourceTypes.getCount());
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerJsonVerifications.java

@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
+
+import com.google.common.collect.Sets;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This test helper class is primarily used by
+ * {@link TestRMWebServicesFairSchedulerCustomResourceTypes}.
+ */
+public class FairSchedulerJsonVerifications {
+
+  private static final Set<String> RESOURCE_FIELDS =
+      Sets.newHashSet("minResources", "amUsedResources", "amMaxResources",
+          "fairResources", "clusterResources", "reservedResources",
+              "maxResources", "usedResources", "steadyFairResources",
+              "demandResources");
+  private final Set<String> customResourceTypes;
+
+  FairSchedulerJsonVerifications(List<String> customResourceTypes) {
+    this.customResourceTypes = Sets.newHashSet(customResourceTypes);
+  }
+
+  public void verify(JSONObject jsonObject) {
+    try {
+      verifyResourcesContainDefaultResourceTypes(jsonObject, RESOURCE_FIELDS);
+      verifyResourcesContainCustomResourceTypes(jsonObject, RESOURCE_FIELDS);
+    } catch (JSONException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private void verifyResourcesContainDefaultResourceTypes(JSONObject queue,
+      Set<String> resourceCategories) throws JSONException {
+    for (String resourceCategory : resourceCategories) {
+      boolean hasResourceCategory = queue.has(resourceCategory);
+      assertTrue("Queue " + queue + " does not have resource category key: "
+          + resourceCategory, hasResourceCategory);
+      verifyResourceContainsDefaultResourceTypes(
+          queue.getJSONObject(resourceCategory));
+    }
+  }
+
+  private void verifyResourceContainsDefaultResourceTypes(
+      JSONObject jsonObject) {
+    Object memory = jsonObject.opt("memory");
+    Object vCores = jsonObject.opt("vCores");
+
+    assertNotNull("Key 'memory' not found in: " + jsonObject, memory);
+    assertNotNull("Key 'vCores' not found in: " + jsonObject, vCores);
+  }
+
+  private void verifyResourcesContainCustomResourceTypes(JSONObject queue,
+      Set<String> resourceCategories) throws JSONException {
+    for (String resourceCategory : resourceCategories) {
+      assertTrue("Queue " + queue + " does not have resource category key: "
+          + resourceCategory, queue.has(resourceCategory));
+      verifyResourceContainsAllCustomResourceTypes(
+          queue.getJSONObject(resourceCategory));
+    }
+  }
+
+  private void verifyResourceContainsAllCustomResourceTypes(
+      JSONObject resourceCategory) throws JSONException {
+    assertTrue("resourceCategory does not have resourceInformations: "
+        + resourceCategory, resourceCategory.has("resourceInformations"));
+
+    JSONObject resourceInformations =
+        resourceCategory.getJSONObject("resourceInformations");
+    assertTrue(
+        "resourceInformations does not have resourceInformation object: "
+            + resourceInformations,
+        resourceInformations.has("resourceInformation"));
+    JSONArray customResources =
+        resourceInformations.getJSONArray("resourceInformation");
+
+    // customResources will include vcores / memory as well
+    assertEquals(
+        "Different number of custom resource types found than expected",
+        customResourceTypes.size(), customResources.length() - 2);
+
+    for (int i = 0; i < customResources.length(); i++) {
+      JSONObject customResource = customResources.getJSONObject(i);
+      assertTrue("Resource type does not have name field: " + customResource,
+          customResource.has("name"));
+      assertTrue("Resource type does not have name resourceType field: "
+          + customResource, customResource.has("resourceType"));
+      assertTrue(
+          "Resource type does not have name units field: " + customResource,
+          customResource.has("units"));
+      assertTrue(
+          "Resource type does not have name value field: " + customResource,
+          customResource.has("value"));
+
+      String name = customResource.getString("name");
+      String unit = customResource.getString("units");
+      String resourceType = customResource.getString("resourceType");
+      Long value = customResource.getLong("value");
+
+      if (ResourceInformation.MEMORY_URI.equals(name)
+          || ResourceInformation.VCORES_URI.equals(name)) {
+        continue;
+      }
+
+      assertTrue("Custom resource type " + name + " not found",
+          customResourceTypes.contains(name));
+      assertEquals("k", unit);
+      assertEquals(ResourceTypes.COUNTABLE,
+          ResourceTypes.valueOf(resourceType));
+      assertNotNull("Custom resource value " + value + " is null!", value);
+    }
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerXmlVerifications.java

@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
+
+
+import com.google.common.collect.Sets;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase.toXml;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlLong;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlString;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This test helper class is primarily used by
+ * {@link TestRMWebServicesFairSchedulerCustomResourceTypes}.
+ */
+public class FairSchedulerXmlVerifications {
+
+  private static final Set<String> RESOURCE_FIELDS = Sets.newHashSet(
+      "minResources", "amUsedResources", "amMaxResources", "fairResources",
+      "clusterResources", "reservedResources", "maxResources", "usedResources",
+      "steadyFairResources", "demandResources");
+  private final Set<String> customResourceTypes;
+
+  FairSchedulerXmlVerifications(List<String> customResourceTypes) {
+    this.customResourceTypes = Sets.newHashSet(customResourceTypes);
+  }
+
+  public void verify(Element element) {
+    verifyResourcesContainDefaultResourceTypes(element, RESOURCE_FIELDS);
+    verifyResourcesContainCustomResourceTypes(element, RESOURCE_FIELDS);
+  }
+
+  private void verifyResourcesContainDefaultResourceTypes(Element queue,
+      Set<String> resourceCategories) {
+    for (String resourceCategory : resourceCategories) {
+      boolean hasResourceCategory = hasChild(queue, resourceCategory);
+      assertTrue("Queue " + queue + " does not have resource category key: "
+          + resourceCategory, hasResourceCategory);
+      verifyResourceContainsDefaultResourceTypes(
+              (Element) queue.getElementsByTagName(resourceCategory).item(0));
+    }
+  }
+
+  private void verifyResourceContainsDefaultResourceTypes(
+      Element element) {
+    Object memory = opt(element, "memory");
+    Object vCores = opt(element, "vCores");
+
+    assertNotNull("Key 'memory' not found in: " + element, memory);
+    assertNotNull("Key 'vCores' not found in: " + element, vCores);
+  }
+
+  private void verifyResourcesContainCustomResourceTypes(Element queue,
+      Set<String> resourceCategories) {
+    for (String resourceCategory : resourceCategories) {
+      assertTrue("Queue " + queue + " does not have key for resourceCategory: "
+          + resourceCategory, hasChild(queue, resourceCategory));
+      verifyResourceContainsCustomResourceTypes(
+              (Element) queue.getElementsByTagName(resourceCategory).item(0));
+    }
+  }
+
+  private void verifyResourceContainsCustomResourceTypes(
+      Element resourceCategory) {
+    assertEquals(
+        toXml(resourceCategory)
+            + " should have only one resourceInformations child!",
+        1, resourceCategory.getElementsByTagName("resourceInformations")
+            .getLength());
+    Element resourceInformations = (Element) resourceCategory
+        .getElementsByTagName("resourceInformations").item(0);
+
+    NodeList customResources =
+        resourceInformations.getElementsByTagName("resourceInformation");
+
+    // customResources will include vcores / memory as well
+    assertEquals(
+        "Different number of custom resource types found than expected",
+        customResourceTypes.size(), customResources.getLength() - 2);
+
+    for (int i = 0; i < customResources.getLength(); i++) {
+      Element customResource = (Element) customResources.item(i);
+      String name = getXmlString(customResource, "name");
+      String unit = getXmlString(customResource, "units");
+      String resourceType = getXmlString(customResource, "resourceType");
+      Long value = getXmlLong(customResource, "value");
+
+      if (ResourceInformation.MEMORY_URI.equals(name)
+          || ResourceInformation.VCORES_URI.equals(name)) {
+        continue;
+      }
+
+      assertTrue("Custom resource type " + name + " not found",
+          customResourceTypes.contains(name));
+      assertEquals("k", unit);
+      assertEquals(ResourceTypes.COUNTABLE,
+          ResourceTypes.valueOf(resourceType));
+      assertNotNull("Resource value should not be null for resource type "
+          + resourceType + ", listing xml contents: " + toXml(customResource),
+          value);
+    }
+  }
+
+  private Object opt(Node node, String child) {
+    NodeList nodes = getElementsByTagNameInternal(node, child);
+    if (nodes.getLength() > 0) {
+      return nodes.item(0);
+    }
+
+    return null;
+  }
+
+  private boolean hasChild(Node node, String child) {
+    return getElementsByTagNameInternal(node, child).getLength() > 0;
+  }
+
+  private NodeList getElementsByTagNameInternal(Node node, String child) {
+    if (node instanceof Element) {
+      return ((Element) node).getElementsByTagName(child);
+    } else if (node instanceof Document) {
+      return ((Document) node).getElementsByTagName(child);
+    } else {
+      throw new IllegalStateException("Unknown type of wrappedObject: " + node
+          + ", type: " + node.getClass());
+    }
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java

@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
+
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.*;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.w3c.dom.Element;
+import javax.ws.rs.core.MediaType;
+import java.lang.reflect.Method;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * This class is to test response representations of queue resources,
+ * explicitly setting custom resource types. with the help of
+ * {@link CustomResourceTypesConfigurationProvider}
+ */
+public class TestRMWebServicesFairSchedulerCustomResourceTypes
+    extends JerseyTestBase {
+  private static MockRM rm;
+  private static YarnConfiguration conf;
+
+  private static class WebServletModule extends ServletModule {
+    @Override
+    protected void configureServlets() {
+      bind(JAXBContextResolver.class);
+      bind(RMWebServices.class);
+      bind(GenericExceptionHandler.class);
+      conf = new YarnConfiguration();
+      conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
+          ResourceScheduler.class);
+      initResourceTypes(conf);
+      rm = new MockRM(conf);
+      bind(ResourceManager.class).toInstance(rm);
+      serve("/*").with(GuiceContainer.class);
+    }
+
+    private void initResourceTypes(YarnConfiguration conf) {
+      conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+          CustomResourceTypesConfigurationProvider.class.getName());
+      ResourceUtils.resetResourceTypes(conf);
+    }
+  }
+
+  @Before
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    createInjectorForWebServletModule();
+  }
+
+  @After
+  public void tearDown() {
+    ResourceUtils.resetResourceTypes(new Configuration());
+  }
+
+  private void createInjectorForWebServletModule() {
+    GuiceServletConfig
+        .setInjector(Guice.createInjector(new WebServletModule()));
+  }
+
+  @After
+  public void teardown() {
+    CustomResourceTypesConfigurationProvider.reset();
+  }
+
+  public TestRMWebServicesFairSchedulerCustomResourceTypes() {
+    super(new WebAppDescriptor.Builder(
+        "org.apache.hadoop.yarn.server.resourcemanager.webapp")
+            .contextListenerClass(GuiceServletConfig.class)
+            .filterClass(com.google.inject.servlet.GuiceFilter.class)
+            .contextPath("jersey-guice-filter").servletPath("/").build());
+  }
+
+  @Test
+  public void testClusterSchedulerWithCustomResourceTypesJson() {
+    FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
+    QueueManager queueManager = scheduler.getQueueManager();
+    // create LeafQueues
+    queueManager.getLeafQueue("root.q.subqueue1", true);
+    queueManager.getLeafQueue("root.q.subqueue2", true);
+
+    FSLeafQueue subqueue1 =
+        queueManager.getLeafQueue("root.q.subqueue1", false);
+    incrementUsedResourcesOnQueue(subqueue1, 33L);
+
+    WebResource path =
+        resource().path("ws").path("v1").path("cluster").path("scheduler");
+    ClientResponse response =
+        path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+
+    verifyJsonResponse(path, response,
+            CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
+  }
+
+  @Test
+  public void testClusterSchedulerWithCustomResourceTypesXml() {
+    FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
+    QueueManager queueManager = scheduler.getQueueManager();
+    // create LeafQueues
+    queueManager.getLeafQueue("root.q.subqueue1", true);
+    queueManager.getLeafQueue("root.q.subqueue2", true);
+
+    FSLeafQueue subqueue1 =
+        queueManager.getLeafQueue("root.q.subqueue1", false);
+    incrementUsedResourcesOnQueue(subqueue1, 33L);
+
+    WebResource path =
+        resource().path("ws").path("v1").path("cluster").path("scheduler");
+    ClientResponse response =
+        path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+
+    verifyXmlResponse(path, response,
+        CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
+  }
+
+  @Test
+  public void testClusterSchedulerWithElevenCustomResourceTypesXml() {
+    CustomResourceTypesConfigurationProvider.setNumberOfResourceTypes(11);
+    createInjectorForWebServletModule();
+
+    FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
+    QueueManager queueManager = scheduler.getQueueManager();
+    // create LeafQueues
+    queueManager.getLeafQueue("root.q.subqueue1", true);
+    queueManager.getLeafQueue("root.q.subqueue2", true);
+
+    FSLeafQueue subqueue1 =
+        queueManager.getLeafQueue("root.q.subqueue1", false);
+    incrementUsedResourcesOnQueue(subqueue1, 33L);
+
+    WebResource path =
+        resource().path("ws").path("v1").path("cluster").path("scheduler");
+    ClientResponse response =
+        path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+
+    verifyXmlResponse(path, response,
+        CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
+  }
+
+  @Test
+  public void testClusterSchedulerElevenWithCustomResourceTypesJson() {
+    CustomResourceTypesConfigurationProvider.setNumberOfResourceTypes(11);
+    createInjectorForWebServletModule();
+
+    FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
+    QueueManager queueManager = scheduler.getQueueManager();
+    // create LeafQueues
+    queueManager.getLeafQueue("root.q.subqueue1", true);
+    queueManager.getLeafQueue("root.q.subqueue2", true);
+
+    FSLeafQueue subqueue1 =
+        queueManager.getLeafQueue("root.q.subqueue1", false);
+    incrementUsedResourcesOnQueue(subqueue1, 33L);
+
+    WebResource path =
+        resource().path("ws").path("v1").path("cluster").path("scheduler");
+    ClientResponse response =
+        path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+
+    verifyJsonResponse(path, response,
+        CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
+  }
+
+  private void verifyJsonResponse(WebResource path, ClientResponse response,
+      List<String> customResourceTypes) {
+    JsonCustomResourceTypeTestcase testCase =
+        new JsonCustomResourceTypeTestcase(path,
+            new BufferedClientResponse(response));
+    testCase.verify(json -> {
+      try {
+        JSONArray queues = json.getJSONObject("scheduler")
+            .getJSONObject("schedulerInfo").getJSONObject("rootQueue")
+            .getJSONObject("childQueues").getJSONArray("queue");
+
+        // childQueueInfo consists of subqueue1 and subqueue2 info
+        assertEquals(2, queues.length());
+        JSONObject firstChildQueue = queues.getJSONObject(0);
+        new FairSchedulerJsonVerifications(customResourceTypes)
+            .verify(firstChildQueue);
+      } catch (JSONException e) {
+        throw new RuntimeException(e);
+      }
+    });
+  }
+
+  private void verifyXmlResponse(WebResource path, ClientResponse response,
+          List<String> customResourceTypes) {
+    XmlCustomResourceTypeTestCase testCase = new XmlCustomResourceTypeTestCase(
+        path, new BufferedClientResponse(response));
+
+    testCase.verify(xml -> {
+      Element scheduler =
+          (Element) xml.getElementsByTagName("scheduler").item(0);
+      Element schedulerInfo =
+          (Element) scheduler.getElementsByTagName("schedulerInfo").item(0);
+      Element rootQueue =
+          (Element) schedulerInfo.getElementsByTagName("rootQueue").item(0);
+
+      Element childQueues =
+          (Element) rootQueue.getElementsByTagName("childQueues").item(0);
+      Element queue =
+          (Element) childQueues.getElementsByTagName("queue").item(0);
+      new FairSchedulerXmlVerifications(customResourceTypes).verify(queue);
+    });
+  }
+
+  private void incrementUsedResourcesOnQueue(final FSLeafQueue queue,
+      final long value) {
+    try {
+      Method incUsedResourceMethod = queue.getClass().getSuperclass()
+          .getDeclaredMethod("incUsedResource", Resource.class);
+      incUsedResourceMethod.setAccessible(true);
+
+      Map<String, Long> customResources =
+          CustomResourceTypesConfigurationProvider.getCustomResourceTypes()
+              .stream()
+              .collect(Collectors.toMap(Function.identity(), v -> value));
+
+      incUsedResourceMethod.invoke(queue,
+          Resource.newInstance(20, 30, customResources));
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoJsonVerifications.java

@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.checkStringEqual;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.checkStringMatch;
+import static org.junit.Assert.*;
+
+/**
+ * Contains all value verifications that are needed to verify {@link AppInfo}
+ * JSON objects.
+ */
+public final class AppInfoJsonVerifications {
+
+  private AppInfoJsonVerifications() {
+    //utility class
+  }
+
+  /**
+   * Tests whether {@link AppInfo} representation object contains the required
+   * values as per defined in the specified app parameter.
+   * @param  app  an RMApp instance that contains the required values
+   *              to test against.
+   */
+  public static void verify(JSONObject info, RMApp app) throws JSONException {
+    checkStringMatch("id", app.getApplicationId().toString(),
+        info.getString("id"));
+    checkStringMatch("user", app.getUser(), info.getString("user"));
+    checkStringMatch("name", app.getName(), info.getString("name"));
+    checkStringMatch("applicationType", app.getApplicationType(),
+        info.getString("applicationType"));
+    checkStringMatch("queue", app.getQueue(), info.getString("queue"));
+    assertEquals("priority doesn't match", 0, info.getInt("priority"));
+    checkStringMatch("state", app.getState().toString(),
+        info.getString("state"));
+    checkStringMatch("finalStatus", app.getFinalApplicationStatus().toString(),
+        info.getString("finalStatus"));
+    assertEquals("progress doesn't match", 0,
+        (float) info.getDouble("progress"), 0.0);
+    if ("UNASSIGNED".equals(info.getString("trackingUI"))) {
+      checkStringMatch("trackingUI", "UNASSIGNED",
+          info.getString("trackingUI"));
+    }
+    checkStringEqual("diagnostics", app.getDiagnostics().toString(),
+        info.getString("diagnostics"));
+    assertEquals("clusterId doesn't match",
+        ResourceManager.getClusterTimeStamp(), info.getLong("clusterId"));
+    assertEquals("startedTime doesn't match", app.getStartTime(),
+        info.getLong("startedTime"));
+    assertEquals("finishedTime doesn't match", app.getFinishTime(),
+        info.getLong("finishedTime"));
+    assertTrue("elapsed time not greater than 0",
+        info.getLong("elapsedTime") > 0);
+    checkStringMatch("amHostHttpAddress",
+        app.getCurrentAppAttempt().getMasterContainer().getNodeHttpAddress(),
+        info.getString("amHostHttpAddress"));
+    assertTrue("amContainerLogs doesn't match",
+        info.getString("amContainerLogs").startsWith("http://"));
+    assertTrue("amContainerLogs doesn't contain user info",
+        info.getString("amContainerLogs").endsWith("/" + app.getUser()));
+    assertEquals("allocatedMB doesn't match", 1024, info.getInt("allocatedMB"));
+    assertEquals("allocatedVCores doesn't match", 1,
+        info.getInt("allocatedVCores"));
+    assertEquals("queueUsagePerc doesn't match", 50.0f,
+        (float) info.getDouble("queueUsagePercentage"), 0.01f);
+    assertEquals("clusterUsagePerc doesn't match", 50.0f,
+        (float) info.getDouble("clusterUsagePercentage"), 0.01f);
+    assertEquals("numContainers doesn't match", 1,
+        info.getInt("runningContainers"));
+    assertNotNull("preemptedResourceSecondsMap should not be null",
+        info.getJSONObject("preemptedResourceSecondsMap"));
+    assertEquals("preemptedResourceMB doesn't match",
+        app.getRMAppMetrics().getResourcePreempted().getMemorySize(),
+        info.getInt("preemptedResourceMB"));
+    assertEquals("preemptedResourceVCores doesn't match",
+        app.getRMAppMetrics().getResourcePreempted().getVirtualCores(),
+        info.getInt("preemptedResourceVCores"));
+    assertEquals("numNonAMContainerPreempted doesn't match",
+        app.getRMAppMetrics().getNumNonAMContainersPreempted(),
+        info.getInt("numNonAMContainerPreempted"));
+    assertEquals("numAMContainerPreempted doesn't match",
+        app.getRMAppMetrics().getNumAMContainersPreempted(),
+        info.getInt("numAMContainerPreempted"));
+    assertEquals("Log aggregation Status doesn't match",
+        app.getLogAggregationStatusForAppReport().toString(),
+        info.getString("logAggregationStatus"));
+    assertEquals("unmanagedApplication doesn't match",
+        app.getApplicationSubmissionContext().getUnmanagedAM(),
+        info.getBoolean("unmanagedApplication"));
+
+    if (app.getApplicationSubmissionContext()
+        .getNodeLabelExpression() != null) {
+      assertEquals("appNodeLabelExpression doesn't match",
+          app.getApplicationSubmissionContext().getNodeLabelExpression(),
+          info.getString("appNodeLabelExpression"));
+    }
+    assertEquals("amNodeLabelExpression doesn't match",
+        app.getAMResourceRequests().get(0).getNodeLabelExpression(),
+        info.getString("amNodeLabelExpression"));
+    assertEquals("amRPCAddress",
+        AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()),
+        info.getString("amRPCAddress"));
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java

@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
+import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
+import org.w3c.dom.Element;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.checkStringMatch;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlBoolean;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlFloat;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlInt;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlLong;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlString;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Contains all value verifications that are needed to verify {@link AppInfo}
+ * XML documents.
+ */
+public final class AppInfoXmlVerifications {
+
+  private AppInfoXmlVerifications() {
+    //utility class
+  }
+
+  /**
+   * Tests whether {@link AppInfo} representation object contains the required
+   * values as per defined in the specified app parameter.
+   * @param info
+   * @param  app  an RMApp instance that contains the required values
+   */
+  public static void verify(Element info, RMApp app) {
+    checkStringMatch("id", app.getApplicationId()
+            .toString(), getXmlString(info, "id"));
+    checkStringMatch("user", app.getUser(),
+            getXmlString(info, "user"));
+    checkStringMatch("name", app.getName(),
+            getXmlString(info, "name"));
+    checkStringMatch("applicationType",
+            app.getApplicationType(), getXmlString(info, "applicationType"));
+    checkStringMatch("queue", app.getQueue(),
+            getXmlString(info, "queue"));
+    assertEquals("priority doesn't match", 0, getXmlInt(info, "priority"));
+    checkStringMatch("state", app.getState().toString(),
+            getXmlString(info, "state"));
+    checkStringMatch("finalStatus", app
+            .getFinalApplicationStatus().toString(),
+            getXmlString(info, "finalStatus"));
+    assertEquals("progress doesn't match", 0, getXmlFloat(info, "progress"),
+        0.0);
+    if ("UNASSIGNED".equals(getXmlString(info, "trackingUI"))) {
+      checkStringMatch("trackingUI", "UNASSIGNED",
+              getXmlString(info, "trackingUI"));
+    }
+    WebServicesTestUtils.checkStringEqual("diagnostics",
+            app.getDiagnostics().toString(), getXmlString(info, "diagnostics"));
+    assertEquals("clusterId doesn't match",
+            ResourceManager.getClusterTimeStamp(),
+            getXmlLong(info, "clusterId"));
+    assertEquals("startedTime doesn't match", app.getStartTime(),
+            getXmlLong(info, "startedTime"));
+    assertEquals("finishedTime doesn't match", app.getFinishTime(),
+            getXmlLong(info, "finishedTime"));
+    assertTrue("elapsed time not greater than 0",
+            getXmlLong(info, "elapsedTime") > 0);
+    checkStringMatch("amHostHttpAddress", app
+                    .getCurrentAppAttempt().getMasterContainer()
+                    .getNodeHttpAddress(),
+            getXmlString(info, "amHostHttpAddress"));
+    assertTrue("amContainerLogs doesn't match",
+        getXmlString(info, "amContainerLogs").startsWith("http://"));
+    assertTrue("amContainerLogs doesn't contain user info",
+        getXmlString(info, "amContainerLogs").endsWith("/" + app.getUser()));
+    assertEquals("allocatedMB doesn't match", 1024,
+            getXmlInt(info, "allocatedMB"));
+    assertEquals("allocatedVCores doesn't match", 1,
+            getXmlInt(info, "allocatedVCores"));
+    assertEquals("queueUsagePerc doesn't match", 50.0f,
+            getXmlFloat(info, "queueUsagePercentage"), 0.01f);
+    assertEquals("clusterUsagePerc doesn't match", 50.0f,
+            getXmlFloat(info, "clusterUsagePercentage"), 0.01f);
+    assertEquals("numContainers doesn't match", 1,
+        getXmlInt(info, "runningContainers"));
+    assertNotNull("preemptedResourceSecondsMap should not be null",
+            info.getElementsByTagName("preemptedResourceSecondsMap"));
+    assertEquals("preemptedResourceMB doesn't match", app
+                    .getRMAppMetrics().getResourcePreempted().getMemorySize(),
+            getXmlInt(info, "preemptedResourceMB"));
+    assertEquals("preemptedResourceVCores doesn't match", app
+                    .getRMAppMetrics().getResourcePreempted().getVirtualCores(),
+            getXmlInt(info, "preemptedResourceVCores"));
+    assertEquals("numNonAMContainerPreempted doesn't match", app
+                    .getRMAppMetrics().getNumNonAMContainersPreempted(),
+            getXmlInt(info, "numNonAMContainerPreempted"));
+    assertEquals("numAMContainerPreempted doesn't match", app
+                    .getRMAppMetrics().getNumAMContainersPreempted(),
+            getXmlInt(info, "numAMContainerPreempted"));
+    assertEquals("Log aggregation Status doesn't match", app
+                    .getLogAggregationStatusForAppReport().toString(),
+            getXmlString(info, "logAggregationStatus"));
+    assertEquals("unmanagedApplication doesn't match", app
+                    .getApplicationSubmissionContext().getUnmanagedAM(),
+            getXmlBoolean(info, "unmanagedApplication"));
+    assertEquals("unmanagedApplication doesn't match",
+            app.getApplicationSubmissionContext().getNodeLabelExpression(),
+            getXmlString(info, "appNodeLabelExpression"));
+    assertEquals("unmanagedApplication doesn't match",
+            app.getAMResourceRequests().get(0).getNodeLabelExpression(),
+            getXmlString(info, "amNodeLabelExpression"));
+    assertEquals("amRPCAddress",
+            AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()),
+            getXmlString(info, "amRPCAddress"));
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+
+import com.sun.jersey.api.client.ClientHandlerException;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.UniformInterfaceException;
+
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
+
+/**
+ * This class is merely a wrapper for {@link ClientResponse}. Given that the
+ * entity input stream of {@link ClientResponse} can be read only once by
+ * default and for some tests it is convenient to read the input stream many
+ * times, this class hides the details of how to do that and prevents
+ * unnecessary code duplication in tests.
+ */
+public class BufferedClientResponse {
+  private ClientResponse response;
+
+  public BufferedClientResponse(ClientResponse response) {
+    response.bufferEntity();
+    this.response = response;
+  }
+
+  public <T> T getEntity(Class<T> clazz)
+          throws ClientHandlerException, UniformInterfaceException {
+    try {
+      response.getEntityInputStream().reset();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    return response.getEntity(clazz);
+  }
+
+  public MediaType getType() {
+    return response.getType();
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java

@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+import com.sun.jersey.api.client.WebResource;
+import org.apache.hadoop.http.JettyUtils;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.MediaType;
+
+import java.util.function.Consumer;
+
+import static org.junit.Assert.*;
+
+/**
+ * This class hides the implementation details of how to verify the structure of
+ * JSON responses. Tests should only provide the path of the
+ * {@link WebResource}, the response from the resource and
+ * the verifier Consumer to
+ * {@link JsonCustomResourceTypeTestcase#verify(Consumer)}. An instance of
+ * {@link JSONObject} will be passed to that consumer to be able to
+ * verify the response.
+ */
+public class JsonCustomResourceTypeTestcase {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(JsonCustomResourceTypeTestcase.class);
+
+  private final WebResource path;
+  private final BufferedClientResponse response;
+  private final JSONObject parsedResponse;
+
+  public JsonCustomResourceTypeTestcase(WebResource path,
+                                        BufferedClientResponse response) {
+    this.path = path;
+    this.response = response;
+    this.parsedResponse = response.getEntity(JSONObject.class);
+  }
+
+  public void verify(Consumer<JSONObject> verifier) {
+    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+        response.getType().toString());
+
+    logResponse();
+
+    String responseStr = response.getEntity(String.class);
+    if (responseStr == null || responseStr.isEmpty()) {
+      throw new IllegalStateException("Response is null or empty!");
+    }
+    verifier.accept(parsedResponse);
+  }
+
+  private void logResponse() {
+    String responseStr = response.getEntity(String.class);
+    LOG.info("Raw response from service URL {}: {}", path.toString(),
+        responseStr);
+    LOG.info("Parsed response from service URL {}: {}", path.toString(),
+        parsedResponse);
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java

@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.util.List;
+import java.util.Map;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Performs value verifications on
+ * {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestInfo}
+ * objects against the values of {@link ResourceRequest}. With the help of the
+ * {@link Builder}, users can also make verifications of the custom resource
+ * types and its values.
+ */
+public class ResourceRequestsJsonVerifications {
+  private final ResourceRequest resourceRequest;
+  private final JSONObject requestInfo;
+  private final Map<String, Long> customResourceTypes;
+  private final List<String> expectedCustomResourceTypes;
+
+  ResourceRequestsJsonVerifications(Builder builder) {
+    this.resourceRequest = builder.resourceRequest;
+    this.requestInfo = builder.requestInfo;
+    this.customResourceTypes = builder.customResourceTypes;
+    this.expectedCustomResourceTypes = builder.expectedCustomResourceTypes;
+  }
+
+  public static void verify(JSONObject requestInfo, ResourceRequest rr)
+      throws JSONException {
+    createDefaultBuilder(requestInfo, rr).build().verify();
+  }
+
+  public static void verifyWithCustomResourceTypes(JSONObject requestInfo,
+      ResourceRequest resourceRequest, List<String> expectedResourceTypes)
+      throws JSONException {
+
+    createDefaultBuilder(requestInfo, resourceRequest)
+        .withExpectedCustomResourceTypes(expectedResourceTypes)
+        .withCustomResourceTypes(
+            extractActualCustomResourceTypes(requestInfo, expectedResourceTypes))
+        .build().verify();
+  }
+
+  private static Builder createDefaultBuilder(JSONObject requestInfo,
+      ResourceRequest resourceRequest) {
+    return new ResourceRequestsJsonVerifications.Builder()
+            .withRequest(resourceRequest)
+            .withRequestInfoJson(requestInfo);
+  }
+
+  private static Map<String, Long> extractActualCustomResourceTypes(
+      JSONObject requestInfo, List<String> expectedResourceTypes)
+      throws JSONException {
+    JSONObject capability = requestInfo.getJSONObject("capability");
+    Map<String, Long> resourceAndValue =
+        extractCustomResorceTypeValues(capability, expectedResourceTypes);
+    Map.Entry<String, Long> resourceEntry =
+        resourceAndValue.entrySet().iterator().next();
+
+    assertTrue(
+        "Found resource type: " + resourceEntry.getKey()
+            + " is not in expected resource types: " + expectedResourceTypes,
+        expectedResourceTypes.contains(resourceEntry.getKey()));
+
+    return resourceAndValue;
+  }
+
+  private static Map<String, Long> extractCustomResorceTypeValues(
+      JSONObject capability, List<String> expectedResourceTypes)
+      throws JSONException {
+    assertTrue(
+        "resourceCategory does not have resourceInformations: " + capability,
+        capability.has("resourceInformations"));
+
+    JSONObject resourceInformations =
+        capability.getJSONObject("resourceInformations");
+    assertTrue(
+        "resourceInformations does not have resourceInformation object: "
+            + resourceInformations,
+        resourceInformations.has("resourceInformation"));
+    JSONArray customResources =
+        resourceInformations.getJSONArray("resourceInformation");
+
+    // customResources will include vcores / memory as well
+    assertEquals(
+        "Different number of custom resource types found than expected",
+        expectedResourceTypes.size(), customResources.length() - 2);
+
+    Map<String, Long> resourceValues = Maps.newHashMap();
+    for (int i = 0; i < customResources.length(); i++) {
+      JSONObject customResource = customResources.getJSONObject(i);
+      assertTrue("Resource type does not have name field: " + customResource,
+          customResource.has("name"));
+      assertTrue("Resource type does not have name resourceType field: "
+          + customResource, customResource.has("resourceType"));
+      assertTrue(
+          "Resource type does not have name units field: " + customResource,
+          customResource.has("units"));
+      assertTrue(
+          "Resource type does not have name value field: " + customResource,
+          customResource.has("value"));
+
+      String name = customResource.getString("name");
+      String unit = customResource.getString("units");
+      String resourceType = customResource.getString("resourceType");
+      Long value = customResource.getLong("value");
+
+      if (ResourceInformation.MEMORY_URI.equals(name)
+          || ResourceInformation.VCORES_URI.equals(name)) {
+        continue;
+      }
+
+      assertTrue("Custom resource type " + name + " not found",
+          expectedResourceTypes.contains(name));
+      assertEquals("k", unit);
+      assertEquals(ResourceTypes.COUNTABLE,
+          ResourceTypes.valueOf(resourceType));
+      assertNotNull("Custom resource value " + value + " is null!", value);
+      resourceValues.put(name, value);
+    }
+
+    return resourceValues;
+  }
+
+  private void verify() throws JSONException {
+    assertEquals("nodeLabelExpression doesn't match",
+        resourceRequest.getNodeLabelExpression(),
+            requestInfo.getString("nodeLabelExpression"));
+    assertEquals("numContainers doesn't match",
+            resourceRequest.getNumContainers(),
+            requestInfo.getInt("numContainers"));
+    assertEquals("relaxLocality doesn't match",
+            resourceRequest.getRelaxLocality(),
+            requestInfo.getBoolean("relaxLocality"));
+    assertEquals("priority does not match",
+            resourceRequest.getPriority().getPriority(),
+            requestInfo.getInt("priority"));
+    assertEquals("resourceName does not match",
+            resourceRequest.getResourceName(),
+            requestInfo.getString("resourceName"));
+    assertEquals("memory does not match",
+        resourceRequest.getCapability().getMemorySize(),
+            requestInfo.getJSONObject("capability").getLong("memory"));
+    assertEquals("vCores does not match",
+        resourceRequest.getCapability().getVirtualCores(),
+            requestInfo.getJSONObject("capability").getLong("vCores"));
+
+    verifyAtLeastOneCustomResourceIsSerialized();
+
+    JSONObject executionTypeRequest =
+            requestInfo.getJSONObject("executionTypeRequest");
+    assertEquals("executionType does not match",
+        resourceRequest.getExecutionTypeRequest().getExecutionType().name(),
+            executionTypeRequest.getString("executionType"));
+    assertEquals("enforceExecutionType does not match",
+            resourceRequest.getExecutionTypeRequest().getEnforceExecutionType(),
+            executionTypeRequest.getBoolean("enforceExecutionType"));
+  }
+
+  /**
+   * JSON serialization produces "invalid JSON" by default as maps are
+   * serialized like this:
+   * "customResources":{"entry":{"key":"customResource-1","value":"0"}}
+   * If the map has multiple keys then multiple entries will be serialized.
+   * Our json parser in tests cannot handle duplicates therefore only one
+   * custom resource will be in the parsed json. See:
+   * https://issues.apache.org/jira/browse/YARN-7505
+   */
+  private void verifyAtLeastOneCustomResourceIsSerialized() {
+    boolean resourceFound = false;
+    for (String expectedCustomResourceType : expectedCustomResourceTypes) {
+      if (customResourceTypes.containsKey(expectedCustomResourceType)) {
+        resourceFound = true;
+        Long resourceValue =
+            customResourceTypes.get(expectedCustomResourceType);
+        assertNotNull("Resource value should not be null!", resourceValue);
+      }
+    }
+    assertTrue("No custom resource type can be found in the response!",
+        resourceFound);
+  }
+
+  /**
+   * Builder class for {@link ResourceRequestsJsonVerifications}.
+   */
+  public static final class Builder {
+    private List<String> expectedCustomResourceTypes = Lists.newArrayList();
+    private Map<String, Long> customResourceTypes;
+    private ResourceRequest resourceRequest;
+    private JSONObject requestInfo;
+
+    Builder() {
+    }
+
+    public static Builder create() {
+      return new Builder();
+    }
+
+    Builder withExpectedCustomResourceTypes(
+            List<String> expectedCustomResourceTypes) {
+      this.expectedCustomResourceTypes = expectedCustomResourceTypes;
+      return this;
+    }
+
+    Builder withCustomResourceTypes(
+            Map<String, Long> customResourceTypes) {
+      this.customResourceTypes = customResourceTypes;
+      return this;
+    }
+
+    Builder withRequest(ResourceRequest resourceRequest) {
+      this.resourceRequest = resourceRequest;
+      return this;
+    }
+
+    Builder withRequestInfoJson(JSONObject requestInfo) {
+      this.requestInfo = requestInfo;
+      return this;
+    }
+
+    public ResourceRequestsJsonVerifications build() {
+      return new ResourceRequestsJsonVerifications(this);
+    }
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java

@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase.toXml;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlBoolean;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlInt;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlLong;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlString;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Performs value verifications on
+ * {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestInfo}
+ * objects against the values of {@link ResourceRequest}. With the help of the
+ * {@link Builder}, users can also make verifications of the custom resource
+ * types and its values.
+ */
+public class ResourceRequestsXmlVerifications {
+  private final ResourceRequest resourceRequest;
+  private final Element requestInfo;
+  private final Map<String, Long> customResourceTypes;
+  private final List<String> expectedCustomResourceTypes;
+
+  ResourceRequestsXmlVerifications(Builder builder) {
+    this.resourceRequest = builder.resourceRequest;
+    this.requestInfo = builder.requestInfo;
+    this.customResourceTypes = builder.customResourceTypes;
+    this.expectedCustomResourceTypes = builder.expectedCustomResourceTypes;
+  }
+
+  public static void verifyWithCustomResourceTypes(Element requestInfo,
+      ResourceRequest resourceRequest, List<String> expectedResourceTypes) {
+
+    createDefaultBuilder(requestInfo, resourceRequest)
+        .withExpectedCustomResourceTypes(expectedResourceTypes)
+        .withCustomResourceTypes(extractActualCustomResourceType(requestInfo,
+            expectedResourceTypes))
+        .build().verify();
+  }
+
+  private static Builder createDefaultBuilder(Element requestInfo,
+      ResourceRequest resourceRequest) {
+    return new ResourceRequestsXmlVerifications.Builder()
+        .withRequest(resourceRequest).withRequestInfo(requestInfo);
+  }
+
+  private static Map<String, Long> extractActualCustomResourceType(
+      Element requestInfo, List<String> expectedResourceTypes) {
+    Element capability =
+        (Element) requestInfo.getElementsByTagName("capability").item(0);
+
+    return extractCustomResorceTypes(capability,
+        Sets.newHashSet(expectedResourceTypes));
+  }
+
+  private static Map<String, Long> extractCustomResorceTypes(Element capability,
+      Set<String> expectedResourceTypes) {
+    assertEquals(
+        toXml(capability) + " should have only one resourceInformations child!",
+        1, capability.getElementsByTagName("resourceInformations").getLength());
+    Element resourceInformations = (Element) capability
+        .getElementsByTagName("resourceInformations").item(0);
+
+    NodeList customResources =
+        resourceInformations.getElementsByTagName("resourceInformation");
+
+    // customResources will include vcores / memory as well
+    assertEquals(
+        "Different number of custom resource types found than expected",
+        expectedResourceTypes.size(), customResources.getLength() - 2);
+
+    Map<String, Long> resourceTypesAndValues = Maps.newHashMap();
+    for (int i = 0; i < customResources.getLength(); i++) {
+      Element customResource = (Element) customResources.item(i);
+      String name = getXmlString(customResource, "name");
+      String unit = getXmlString(customResource, "units");
+      String resourceType = getXmlString(customResource, "resourceType");
+      Long value = getXmlLong(customResource, "value");
+
+      if (ResourceInformation.MEMORY_URI.equals(name)
+          || ResourceInformation.VCORES_URI.equals(name)) {
+        continue;
+      }
+
+      assertTrue("Custom resource type " + name + " not found",
+          expectedResourceTypes.contains(name));
+      assertEquals("k", unit);
+      assertEquals(ResourceTypes.COUNTABLE,
+          ResourceTypes.valueOf(resourceType));
+      assertNotNull("Resource value should not be null for resource type "
+          + resourceType + ", listing xml contents: " + toXml(customResource),
+          value);
+      resourceTypesAndValues.put(name, value);
+    }
+
+    return resourceTypesAndValues;
+  }
+
+  private void verify() {
+    assertEquals("nodeLabelExpression doesn't match",
+        resourceRequest.getNodeLabelExpression(),
+        getXmlString(requestInfo, "nodeLabelExpression"));
+    assertEquals("numContainers doesn't match",
+        resourceRequest.getNumContainers(),
+        getXmlInt(requestInfo, "numContainers"));
+    assertEquals("relaxLocality doesn't match",
+        resourceRequest.getRelaxLocality(),
+        getXmlBoolean(requestInfo, "relaxLocality"));
+    assertEquals("priority does not match",
+        resourceRequest.getPriority().getPriority(),
+        getXmlInt(requestInfo, "priority"));
+    assertEquals("resourceName does not match",
+        resourceRequest.getResourceName(),
+        getXmlString(requestInfo, "resourceName"));
+    Element capability = (Element) requestInfo
+            .getElementsByTagName("capability").item(0);
+    assertEquals("memory does not match",
+        resourceRequest.getCapability().getMemorySize(),
+        getXmlLong(capability, "memory"));
+    assertEquals("vCores does not match",
+        resourceRequest.getCapability().getVirtualCores(),
+        getXmlLong(capability, "vCores"));
+
+    for (String expectedCustomResourceType : expectedCustomResourceTypes) {
+      assertTrue(
+          "Custom resource type " + expectedCustomResourceType
+              + " cannot be found!",
+          customResourceTypes.containsKey(expectedCustomResourceType));
+
+      Long resourceValue = customResourceTypes.get(expectedCustomResourceType);
+      assertNotNull("Resource value should not be null!", resourceValue);
+    }
+
+    Element executionTypeRequest = (Element) requestInfo
+        .getElementsByTagName("executionTypeRequest").item(0);
+    assertEquals("executionType does not match",
+        resourceRequest.getExecutionTypeRequest().getExecutionType().name(),
+        getXmlString(executionTypeRequest, "executionType"));
+    assertEquals("enforceExecutionType does not match",
+        resourceRequest.getExecutionTypeRequest().getEnforceExecutionType(),
+        getXmlBoolean(executionTypeRequest, "enforceExecutionType"));
+  }
+
+  /**
+   * Builder class for {@link ResourceRequestsXmlVerifications}.
+   */
+  public static final class Builder {
+    private List<String> expectedCustomResourceTypes = Lists.newArrayList();
+    private Map<String, Long> customResourceTypes;
+    private ResourceRequest resourceRequest;
+    private Element requestInfo;
+
+    Builder() {
+    }
+
+    public static Builder create() {
+      return new Builder();
+    }
+
+    Builder withExpectedCustomResourceTypes(
+        List<String> expectedCustomResourceTypes) {
+      this.expectedCustomResourceTypes = expectedCustomResourceTypes;
+      return this;
+    }
+
+    Builder withCustomResourceTypes(Map<String, Long> customResourceTypes) {
+      this.customResourceTypes = customResourceTypes;
+      return this;
+    }
+
+    Builder withRequest(ResourceRequest resourceRequest) {
+      this.resourceRequest = resourceRequest;
+      return this;
+    }
+
+    Builder withRequestInfo(Element requestInfo) {
+      this.requestInfo = requestInfo;
+      return this;
+    }
+
+    public ResourceRequestsXmlVerifications build() {
+      return new ResourceRequestsXmlVerifications(this);
+    }
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java

@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+import com.sun.jersey.api.client.WebResource;
+import org.apache.hadoop.http.JettyUtils;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import org.xml.sax.InputSource;
+
+import javax.ws.rs.core.MediaType;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.*;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.util.function.Consumer;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * This class hides the implementation details of how to verify the structure of
+ * XML responses. Tests should only provide the path of the
+ * {@link WebResource}, the response from the resource and
+ * the verifier Consumer to
+ * {@link XmlCustomResourceTypeTestCase#verify(Consumer)}. An instance of
+ * {@link JSONObject} will be passed to that consumer to be able to
+ * verify the response.
+ */
+public class XmlCustomResourceTypeTestCase {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(XmlCustomResourceTypeTestCase.class);
+
+  private WebResource path;
+  private BufferedClientResponse response;
+  private Document parsedResponse;
+
+  public XmlCustomResourceTypeTestCase(WebResource path,
+                                       BufferedClientResponse response) {
+    this.path = path;
+    this.response = response;
+  }
+
+  public void verify(Consumer<Document> verifier) {
+    assertEquals(MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8,
+        response.getType().toString());
+
+    parsedResponse = parseXml(response);
+    logResponse(parsedResponse);
+    verifier.accept(parsedResponse);
+  }
+
+  private Document parseXml(BufferedClientResponse response) {
+    try {
+      String xml = response.getEntity(String.class);
+      DocumentBuilder db =
+          DocumentBuilderFactory.newInstance().newDocumentBuilder();
+      InputSource is = new InputSource();
+      is.setCharacterStream(new StringReader(xml));
+
+      return db.parse(is);
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private void logResponse(Document doc) {
+    String responseStr = response.getEntity(String.class);
+    LOG.info("Raw response from service URL {}: {}", path.toString(),
+        responseStr);
+    LOG.info("Parsed response from service URL {}: {}", path.toString(),
+        toXml(doc));
+  }
+
+  public static String toXml(Node node) {
+    StringWriter writer;
+    try {
+      TransformerFactory tf = TransformerFactory.newInstance();
+      Transformer transformer = tf.newTransformer();
+      transformer.setOutputProperty(OutputKeys.INDENT, "yes");
+      transformer.setOutputProperty(
+          "{http://xml.apache.org/xslt}indent" + "-amount", "2");
+      writer = new StringWriter();
+      transformer.transform(new DOMSource(node), new StreamResult(writer));
+    } catch (TransformerException e) {
+      throw new RuntimeException(e);
+    }
+
+    return writer.getBuffer().toString();
+  }
+}