YARN-11538. CS UI: queue filter do not work as expected when submitti… (#5890)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java

@@ -889,6 +889,16 @@ public class ClientRMService extends AbstractService implements
     final Set<ApplicationId> runningAppsFilteredByQueues =
         getRunningAppsFilteredByQueues(apps, queues);
 
+    Set<String> queuePaths = new HashSet<>();
+    for (String queue : queues) {
+      String queuePath = rmAppManager.getQueuePath(queue);
+      if (queuePath != null) {
+        queuePaths.add(queuePath);
+      } else {
+        queuePaths.add(queue);
+      }
+    }
+
     Iterator<RMApp> appsIter = apps.values().iterator();
     
     List<ApplicationReport> reports = new ArrayList<ApplicationReport>();
@@ -901,9 +911,9 @@ public class ClientRMService extends AbstractService implements
         continue;
       }
 
-      if (queues != null && !queues.isEmpty()) {
+      if (queuePaths != null && !queuePaths.isEmpty()) {
         if (!runningAppsFilteredByQueues.contains(application.getApplicationId()) &&
-            !queues.contains(application.getQueue())) {
+            !queuePaths.contains(application.getQueue())) {
           continue;
         }
       }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.yarn.server.resourcemanager;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
@@ -28,6 +29,7 @@ import java.util.concurrent.Future;
 
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.conf.HAUtil;
 import org.apache.hadoop.yarn.security.ConfiguredYarnAuthorizer;
 import org.apache.hadoop.yarn.security.Permission;
@@ -552,6 +554,9 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
       }
     }
 
+    // Get full queue path for the application when submitting by short queue name
+    placementQueueName = getQueuePath(placementQueueName);
+
     // Create RMApp
     RMAppImpl application =
         new RMAppImpl(applicationId, rmContext, this.conf,
@@ -582,6 +587,20 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
     return application;
   }
 
+  public String getQueuePath(String queueName) {
+    String queuePath = queueName;
+    try {
+      QueueInfo queueInfo =
+          scheduler.getQueueInfo(queueName, false, false);
+      if (queueInfo != null && queueInfo.getQueuePath() != null) {
+        queuePath = queueInfo.getQueuePath();
+      }
+    } catch (IOException e) {
+      // if the queue does not exist, we just ignore here
+    }
+    return queuePath;
+  }
+
   private boolean checkPermission(AccessRequest accessRequest,
                                   YarnAuthorizationProvider dynamicAuthorizer) {
     return authorizer.checkPermission(accessRequest) ||

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java

@@ -2725,7 +2725,7 @@ public class CapacityScheduler extends
       application.setQueue(dest);
       LOG.info("App: " + appId + " successfully moved from " + sourceQueueName
           + " to: " + destQueueName);
-      return targetQueueName;
+      return dest.getQueuePath();
     } finally {
       writeLock.unlock();
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java

@@ -355,6 +355,33 @@ public class TestAppManager extends AppManagerTestBase{
         "test1");
   }
 
+  @Test
+  public void testQueueSubmitWithLeafQueueName()
+      throws YarnException {
+    YarnConfiguration conf = createYarnACLEnabledConfiguration();
+    CapacitySchedulerConfiguration csConf = new
+        CapacitySchedulerConfiguration(conf, false);
+    csConf.set(PREFIX + "root.queues", "default,test");
+
+    csConf.setCapacity("root.default", 50.0f);
+    csConf.setMaximumCapacity("root.default", 100.0f);
+
+    csConf.setCapacity("root.test", 50.0f);
+    csConf.setMaximumCapacity("root.test", 100.0f);
+
+    MockRM newMockRM = new MockRM(csConf);
+    RMContext newMockRMContext = newMockRM.getRMContext();
+    TestRMAppManager newAppMonitor = createAppManager(newMockRMContext, conf);
+
+    ApplicationSubmissionContext submission = createAppSubmissionContext(MockApps.newAppID(1));
+    submission.setQueue("test");
+    verifyAppSubmission(submission,
+        newAppMonitor,
+        newMockRMContext,
+        "test",
+        "root.test");
+  }
+
   @Test
   public void testQueueSubmitWithACLsEnabledWithQueueMappingForLegacyAutoCreatedQueue()
       throws IOException, YarnException {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java

@@ -75,7 +75,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerStat
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
@@ -1168,7 +1167,7 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
       //verify queue name when rmContainer is recovered
       if (scheduler instanceof CapacityScheduler) {
         Assert.assertEquals(
-            CapacitySchedulerConfiguration.ROOT + "." + app1.getQueue(),
+            app1.getQueue(),
             rmContainer.getQueueName());
       } else {
         Assert.assertEquals(app1.getQueue(), rmContainer.getQueueName());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java

@@ -2041,13 +2041,11 @@ public class TestRMWebServicesApps extends JerseyTestBase {
         appIds.contains(runningApp1.getApplicationId().toString()));
     assertTrue("Running app 2 should be in the result list!",
         appIds.contains(runningApp2.getApplicationId().toString()));
-    assertFalse("Finished app 1 should not be in the result list " +
-            "as it was submitted to 'root.default' but the query is for 'default'",
+    assertTrue("Running app 1 should be in the result list!",
         appIds.contains(finishedApp1.getApplicationId().toString()));
-    assertTrue("Finished app 2 should be in the result list " +
-            "as it was submitted to 'default' and the query is exactly for 'default'",
+    assertTrue("Running app 1 should be in the result list!",
         appIds.contains(finishedApp2.getApplicationId().toString()));
-    assertEquals("incorrect number of elements", 3, array.length());
+    assertEquals("incorrect number of elements", 4, array.length());
 
     rm.stop();
   }
@@ -2110,13 +2108,11 @@ public class TestRMWebServicesApps extends JerseyTestBase {
         appIds.contains(runningApp1.getApplicationId().toString()));
     assertTrue("Running app 2 should be in the result list!",
         appIds.contains(runningApp2.getApplicationId().toString()));
-    assertTrue("Finished app 1 should be in the result list, " +
-            "as it was submitted to 'root.default' and the query is exactly for 'root.default'!",
+    assertTrue("Running app 2 should be in the result list!",
         appIds.contains(finishedApp1.getApplicationId().toString()));
-    assertFalse("Finished app 2 should not be in the result list, " +
-            "as it was submitted to 'default' but the query is for 'root.default'!",
+    assertTrue("Running app 2 should be in the result list!",
         appIds.contains(finishedApp2.getApplicationId().toString()));
-    assertEquals("incorrect number of elements", 3, array.length());
+    assertEquals("incorrect number of elements", 4, array.length());
 
     rm.stop();
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java

@@ -1066,7 +1066,7 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
             .constructWebResource("apps", app.getApplicationId().toString(),
               "queue").accept(contentType).get(ClientResponse.class);
       assertResponseStatusCode(Status.OK, response.getStatusInfo());
-      String expectedQueue = "default";
+      String expectedQueue = "root.default";
       if(!isCapacityScheduler) {
         expectedQueue = "root." + webserviceUserName;
       }
@@ -1230,10 +1230,7 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
           continue;
         }
         assertResponseStatusCode(Status.OK, response.getStatusInfo());
-        String expectedQueue = "test";
-        if(!isCapacityScheduler) {
-          expectedQueue = "root.test";
-        }
+        String expectedQueue = "root.test";
         if (mediaType.contains(MediaType.APPLICATION_JSON)) {
           verifyAppQueueJson(response, expectedQueue);
         } else {
@@ -1256,7 +1253,7 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
               .put(ClientResponse.class);
         assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo());
         if(isCapacityScheduler) {
-          Assert.assertEquals("default", app.getQueue());
+          Assert.assertEquals("root.default", app.getQueue());
         }
         else {
           Assert.assertEquals("root.someuser", app.getQueue());