YARN-10274. Merge QueueMapping and QueueMappingEntity. Contributed by Gergely Pollak

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java

@@ -48,7 +48,7 @@ public class AppNameMappingPlacementRule extends PlacementRule {
   private static final String QUEUE_MAPPING_NAME = "app-name";
 
   private boolean overrideWithQueueMappings = false;
-  private List<QueueMappingEntity> mappings = null;
+  private List<QueueMapping> mappings = null;
   protected CapacitySchedulerQueueManager queueManager;
 
   public AppNameMappingPlacementRule() {
@@ -56,7 +56,7 @@ public class AppNameMappingPlacementRule extends PlacementRule {
   }
 
   public AppNameMappingPlacementRule(boolean overrideWithQueueMappings,
-      List<QueueMappingEntity> newMappings) {
+      List<QueueMapping> newMappings) {
     this.overrideWithQueueMappings = overrideWithQueueMappings;
     this.mappings = newMappings;
   }
@@ -76,16 +76,16 @@ public class AppNameMappingPlacementRule extends PlacementRule {
     LOG.info(
         "Initialized App Name queue mappings, override: " + overrideWithQueueMappings);
 
-    List<QueueMappingEntity> queueMappings =
+    List<QueueMapping> queueMappings =
         conf.getQueueMappingEntity(QUEUE_MAPPING_NAME);
 
     // Get new user mappings
-    List<QueueMappingEntity> newMappings = new ArrayList<>();
+    List<QueueMapping> newMappings = new ArrayList<>();
 
     queueManager = schedulerContext.getCapacitySchedulerQueueManager();
 
     // check if mappings refer to valid queues
-    for (QueueMappingEntity mapping : queueMappings) {
+    for (QueueMapping mapping : queueMappings) {
       QueuePath queuePath = mapping.getQueuePath();
 
       if (isStaticQueueMapping(mapping)) {
@@ -109,7 +109,7 @@ public class AppNameMappingPlacementRule extends PlacementRule {
           //validate if parent queue is specified,
           // then it should exist and
           // be an instance of AutoCreateEnabledParentQueue
-          QueueMappingEntity newMapping =
+          QueueMapping newMapping =
               validateAndGetAutoCreatedQueueMapping(queueManager, mapping,
                   queuePath);
           if (newMapping == null) {
@@ -123,7 +123,7 @@ public class AppNameMappingPlacementRule extends PlacementRule {
           //   if its an instance of leaf queue
           //   if its an instance of auto created leaf queue,
           // then extract parent queue name and update queue mapping
-          QueueMappingEntity newMapping = validateAndGetQueueMapping(
+          QueueMapping newMapping = validateAndGetQueueMapping(
               queueManager, queue, mapping, queuePath);
           newMappings.add(newMapping);
         }
@@ -134,7 +134,7 @@ public class AppNameMappingPlacementRule extends PlacementRule {
         // if parent queue is specified, then
         //  parent queue exists and an instance of AutoCreateEnabledParentQueue
         //
-        QueueMappingEntity newMapping = validateAndGetAutoCreatedQueueMapping(
+        QueueMapping newMapping = validateAndGetAutoCreatedQueueMapping(
             queueManager, mapping, queuePath);
         if (newMapping != null) {
           newMappings.add(newMapping);
@@ -160,7 +160,7 @@ public class AppNameMappingPlacementRule extends PlacementRule {
 
   private ApplicationPlacementContext getAppPlacementContext(String user,
       String applicationName) throws IOException {
-    for (QueueMappingEntity mapping : mappings) {
+    for (QueueMapping mapping : mappings) {
       if (mapping.getSource().equals(CURRENT_APP_MAPPING)) {
         if (mapping.getQueue().equals(CURRENT_APP_MAPPING)) {
           return getPlacementContext(mapping, applicationName, queueManager);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMapping.java

@@ -82,6 +82,7 @@ public class QueueMapping {
     this.source = builder.source;
     this.queue = builder.queue;
     this.parentQueue = builder.parentQueue;
+    this.fullPath = (parentQueue != null) ? (parentQueue + DOT + queue) : queue;
   }
 
   /**
@@ -89,8 +90,9 @@ public class QueueMapping {
    *
    */
   public enum MappingType {
-
-    USER("u"), GROUP("g");
+    USER("u"),
+    GROUP("g"),
+    APPLICATION("a");
 
     private final String type;
 
@@ -108,6 +110,7 @@ public class QueueMapping {
   private String source;
   private String queue;
   private String parentQueue;
+  private String fullPath;
 
   private final static String DELIMITER = ":";
 
@@ -132,7 +135,7 @@ public class QueueMapping {
   }
 
   public String getFullPath() {
-    return (parentQueue != null ? parentQueue + DOT + queue : queue);
+    return fullPath;
   }
 
   public QueuePath getQueuePath() {
@@ -197,4 +200,10 @@ public class QueueMapping {
     return type.toString() + DELIMITER + source + DELIMITER
         + (parentQueue != null ? parentQueue + "." + queue : queue);
   }
+
+  public String toTypelessString() {
+    return source + DELIMITER
+        + (parentQueue != null ? parentQueue + "." + queue : queue);
+  }
+
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMappingEntity.java

@@ -1,98 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.placement;
-
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
-
-public class QueueMappingEntity {
-  private String source;
-  private String queue;
-  private String parentQueue;
-  private String fullPath;
-
-  public final static String DELIMITER = ":";
-
-  public QueueMappingEntity(String source, String queue) {
-    this.source = source;
-    this.queue = queue;
-    this.parentQueue = null;
-    this.fullPath = queue;
-  }
-  public QueueMappingEntity(String source, String queue, String parentQueue) {
-    this.source = source;
-    this.queue = queue;
-    this.parentQueue = parentQueue;
-    this.fullPath = parentQueue + DOT + queue;
-  }
-
-  public QueueMappingEntity(String source, QueuePath path) {
-    this.source = source;
-    this.queue = path.getLeafQueue();
-    this.parentQueue = path.getParentQueue();
-    this.fullPath = parentQueue + DOT + queue;
-  }
-
-  public String getQueue() {
-    return queue;
-  }
-
-  public String getParentQueue() {
-    return parentQueue;
-  }
-
-  public String getFullPath() {
-    return fullPath;
-  }
-
-  public String getSource() {
-    return source;
-  }
-
-  public boolean hasParentQueue() {
-    return parentQueue != null;
-  }
-
-  public QueuePath getQueuePath() {
-    //This is to make sure the parsing is the same everywhere, but the
-    //whole parsing part should be moved to QueuePathConstructor
-    return QueuePlacementRuleUtils.extractQueuePath(getFullPath());
-  }
-
-  @Override
-  public int hashCode() {
-    return super.hashCode();
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (obj instanceof QueueMappingEntity) {
-      QueueMappingEntity other = (QueueMappingEntity) obj;
-      return (other.source.equals(source) &&
-          other.queue.equals(queue));
-    } else {
-      return false;
-    }
-  }
-
-  public String toString() {
-    return source + DELIMITER + (parentQueue != null ?
-        parentQueue + DOT + queue :
-        queue);
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java

@@ -65,8 +65,8 @@ public final class QueuePlacementRuleUtils {
     }
   }
 
-  public static QueueMappingEntity validateAndGetAutoCreatedQueueMapping(
-      CapacitySchedulerQueueManager queueManager, QueueMappingEntity mapping,
+  public static QueueMapping validateAndGetAutoCreatedQueueMapping(
+      CapacitySchedulerQueueManager queueManager, QueueMapping mapping,
       QueuePath queuePath) throws IOException {
     if (queuePath.hasParentQueue()) {
       //if parent queue is specified,
@@ -74,16 +74,19 @@ public final class QueuePlacementRuleUtils {
       validateQueueMappingUnderParentQueue(queueManager.getQueue(
           queuePath.getParentQueue()), queuePath.getParentQueue(),
           queuePath.getFullPath());
-      return new QueueMappingEntity(mapping.getSource(),
-          queuePath.getFullPath(), queuePath.getParentQueue());
+      return QueueMapping.QueueMappingBuilder.create()
+          .type(mapping.getType())
+          .source(mapping.getSource())
+          .queuePath(queuePath)
+          .build();
     }
 
     return null;
   }
 
-  public static QueueMappingEntity validateAndGetQueueMapping(
+  public static QueueMapping validateAndGetQueueMapping(
       CapacitySchedulerQueueManager queueManager, CSQueue queue,
-      QueueMappingEntity mapping, QueuePath queuePath) throws IOException {
+      QueueMapping mapping, QueuePath queuePath) throws IOException {
     if (!(queue instanceof LeafQueue)) {
       throw new IOException(
           "mapping contains invalid or non-leaf queue : " +
@@ -93,7 +96,7 @@ public final class QueuePlacementRuleUtils {
     if (queue instanceof AutoCreatedLeafQueue && queue
         .getParent() instanceof ManagedParentQueue) {
 
-      QueueMappingEntity newMapping = validateAndGetAutoCreatedQueueMapping(
+      QueueMapping newMapping = validateAndGetAutoCreatedQueueMapping(
           queueManager, mapping, queuePath);
       if (newMapping == null) {
         throw new IOException(
@@ -105,7 +108,7 @@ public final class QueuePlacementRuleUtils {
     return mapping;
   }
 
-  public static boolean isStaticQueueMapping(QueueMappingEntity mapping) {
+  public static boolean isStaticQueueMapping(QueueMapping mapping) {
     return !mapping.getQueue().contains(CURRENT_USER_MAPPING) && !mapping
         .getQueue().contains(PRIMARY_GROUP_MAPPING)
         && !mapping.getQueue().contains(SECONDARY_GROUP_MAPPING);
@@ -126,13 +129,13 @@ public final class QueuePlacementRuleUtils {
   }
 
   public static ApplicationPlacementContext getPlacementContext(
-      QueueMappingEntity mapping, CapacitySchedulerQueueManager queueManager)
+      QueueMapping mapping, CapacitySchedulerQueueManager queueManager)
       throws IOException {
     return getPlacementContext(mapping, mapping.getQueue(), queueManager);
   }
 
   public static ApplicationPlacementContext getPlacementContext(
-      QueueMappingEntity mapping, String leafQueueName,
+      QueueMapping mapping, String leafQueueName,
       CapacitySchedulerQueueManager queueManager) throws IOException {
 
     //leafQueue name no longer identifies a queue uniquely checking ambiguity

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java

@@ -42,7 +42,6 @@ import org.apache.hadoop.yarn.security.AccessType;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
-import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMappingEntity;
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AppPriorityACLConfigurationParser.AppPriorityACLKeyType;
@@ -1039,12 +1038,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
     setBoolean(ENABLE_QUEUE_MAPPING_OVERRIDE, overrideWithQueueMappings);
   }
 
-  public List<QueueMappingEntity> getQueueMappingEntity(
+  public List<QueueMapping> getQueueMappingEntity(
       String queueMappingSuffix) {
     String queueMappingName = buildQueueMappingRuleProperty(queueMappingSuffix);
 
-    List<QueueMappingEntity> mappings =
-        new ArrayList<QueueMappingEntity>();
+    List<QueueMapping> mappings =
+        new ArrayList<QueueMapping>();
     Collection<String> mappingsString =
         getTrimmedStringCollection(queueMappingName);
     for (String mappingValue : mappingsString) {
@@ -1058,10 +1057,11 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
 
       //Mappings should be consistent, and have the parent path parsed
       // from the beginning
-      QueueMappingEntity m = new QueueMappingEntity(
-          mapping[0],
-          QueuePlacementRuleUtils.extractQueuePath(mapping[1]));
-
+      QueueMapping m = QueueMapping.QueueMappingBuilder.create()
+          .type(QueueMapping.MappingType.APPLICATION)
+          .source(mapping[0])
+          .queuePath(QueuePlacementRuleUtils.extractQueuePath(mapping[1]))
+          .build();
       mappings.add(m);
     }
 
@@ -1076,15 +1076,15 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
   }
 
   @VisibleForTesting
-  public void setQueueMappingEntities(List<QueueMappingEntity> queueMappings,
+  public void setQueueMappingEntities(List<QueueMapping> queueMappings,
       String queueMappingSuffix) {
     if (queueMappings == null) {
       return;
     }
 
     List<String> queueMappingStrs = new ArrayList<>();
-    for (QueueMappingEntity mapping : queueMappings) {
-      queueMappingStrs.add(mapping.toString());
+    for (QueueMapping mapping : queueMappings) {
+      queueMappingStrs.add(mapping.toTypelessString());
     }
 
     String mappingRuleProp = buildQueueMappingRuleProperty(queueMappingSuffix);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java

@@ -47,13 +47,13 @@ public class TestAppNameMappingPlacementRule {
         SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
   }
 
-  private void verifyQueueMapping(QueueMappingEntity queueMapping,
+  private void verifyQueueMapping(QueueMapping queueMapping,
       String user, String expectedQueue) throws YarnException {
     verifyQueueMapping(queueMapping, user,
         queueMapping.getQueue(), expectedQueue, false);
   }
 
-  private void verifyQueueMapping(QueueMappingEntity queueMapping,
+  private void verifyQueueMapping(QueueMapping queueMapping,
       String user, String inputQueue, String expectedQueue,
       boolean overwrite) throws YarnException {
     AppNameMappingPlacementRule rule = new AppNameMappingPlacementRule(
@@ -81,23 +81,31 @@ public class TestAppNameMappingPlacementRule {
         ctx != null ? ctx.getQueue() : inputQueue);
   }
 
+  public QueueMapping queueMappingBuilder(String source, String queue) {
+    return QueueMapping.QueueMappingBuilder.create()
+        .type(QueueMapping.MappingType.APPLICATION)
+        .source(source)
+        .queue(queue)
+        .build();
+  }
+
   @Test
   public void testMapping() throws YarnException {
     // simple base case for mapping user to queue
-    verifyQueueMapping(new QueueMappingEntity(APP_NAME,
+    verifyQueueMapping(queueMappingBuilder(APP_NAME,
         "q1"), "user_1", "q1");
-    verifyQueueMapping(new QueueMappingEntity("%application", "q2"), "user_1",
+    verifyQueueMapping(queueMappingBuilder("%application", "q2"), "user_1",
         "q2");
-    verifyQueueMapping(new QueueMappingEntity("%application", "%application"),
+    verifyQueueMapping(queueMappingBuilder("%application", "%application"),
         "user_1", APP_NAME);
 
     // specify overwritten, and see if user specified a queue, and it will be
     // overridden
-    verifyQueueMapping(new QueueMappingEntity(APP_NAME,
+    verifyQueueMapping(queueMappingBuilder(APP_NAME,
         "q1"), "1", "q2", "q1", true);
 
     // if overwritten not specified, it should be which user specified
-    verifyQueueMapping(new QueueMappingEntity(APP_NAME,
+    verifyQueueMapping(queueMappingBuilder(APP_NAME,
             "q1"), "1", "q2", "q2", false);
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementManager.java

@@ -95,8 +95,12 @@ public class TestPlacementManager {
 
     Assert.assertNull("Placement should be null",
         pm.placeApplication(asc, USER2));
-    QueueMappingEntity queueMappingEntity = new QueueMappingEntity(APP_NAME,
-        USER1, PARENT_QUEUE);
+    QueueMapping queueMappingEntity = QueueMapping.QueueMappingBuilder.create()
+      .type(MappingType.APPLICATION)
+      .source(APP_NAME)
+      .queue(USER1)
+      .parentQueue(PARENT_QUEUE)
+      .build();
 
     AppNameMappingPlacementRule anRule = new AppNameMappingPlacementRule(false,
         Arrays.asList(queueMappingEntity));

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java

@@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
-import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMappingEntity;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SimpleGroupsMapping;
@@ -84,17 +83,20 @@ public class TestCapacitySchedulerQueueMappingFactory {
     existingMappingsForUG.addAll(queueMappingsForUG);
     conf.setQueueMappings(existingMappingsForUG);
 
-    List<QueueMappingEntity> existingMappingsForAN =
+    List<QueueMapping> existingMappingsForAN =
         conf.getQueueMappingEntity(QUEUE_MAPPING_NAME);
 
     //set queue mapping
-    List<QueueMappingEntity> queueMappingsForAN =
+    List<QueueMapping> queueMappingsForAN =
         new ArrayList<>();
     for (int i = 0; i < sourceIds.length; i++) {
       //Set C as parent queue name for auto queue creation
-      QueueMappingEntity queueMapping =
-          new QueueMappingEntity(USER + sourceIds[i],
-              getQueueMapping(parentQueue, USER + sourceIds[i]));
+      QueueMapping queueMapping = QueueMapping.QueueMappingBuilder.create()
+          .type(MappingType.APPLICATION)
+          .source(USER + sourceIds[i])
+          .queue(getQueueMapping(parentQueue, USER + sourceIds[i]))
+          .build();
+
       queueMappingsForAN.add(queueMapping);
     }