Browse Source

YARN-2800. Remove MemoryNodeLabelsStore and add a way to enable/disable node labels feature. Contributed by Wangda Tan.

(cherry picked from commit 24aa462673d392fed859f8088acf9679ae62a129)
Tsuyoshi Ozawa 10 years ago
parent
commit
25f2cf8a4e
16 changed files with 152 additions and 37 deletions
  1. 3 0
      hadoop-yarn-project/CHANGES.txt
  2. 10 6
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
  3. 1 0
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
  4. 41 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
  5. 60 2
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
  6. 1 0
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
  7. 1 5
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
  8. 2 2
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
  9. 1 0
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
  10. 9 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java
  11. 6 3
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
  12. 3 3
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
  13. 2 2
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
  14. 3 3
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
  15. 6 6
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
  16. 3 3
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java

+ 3 - 0
hadoop-yarn-project/CHANGES.txt

@@ -163,6 +163,9 @@ Release 2.7.0 - UNRELEASED
 
 
     YARN-2984. Metrics for container's actual memory usage. (kasha)
     YARN-2984. Metrics for container's actual memory usage. (kasha)
 
 
+    YARN-2800. Remove MemoryNodeLabelsStore and add a way to enable/disable
+    node labels feature. (Wangda Tan via ozawa)
+
   OPTIMIZATIONS
   OPTIMIZATIONS
 
 
   BUG FIXES
   BUG FIXES

+ 10 - 6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -1648,14 +1648,10 @@ public class YarnConfiguration extends Configuration {
   public static final String YARN_HTTP_POLICY_DEFAULT = HttpConfig.Policy.HTTP_ONLY
   public static final String YARN_HTTP_POLICY_DEFAULT = HttpConfig.Policy.HTTP_ONLY
       .name();
       .name();
   
   
-  public static final String NODE_LABELS_PREFIX = YARN_PREFIX + "node-labels.";
-
   /**
   /**
-   * Class for RMNodeLabelsManager Please note this value should be consistent
-   * in client nodes and RM node(s)
+   * Node-labels configurations
    */
    */
-  public static final String RM_NODE_LABELS_MANAGER_CLASS = NODE_LABELS_PREFIX
-      + "manager-class";
+  public static final String NODE_LABELS_PREFIX = YARN_PREFIX + "node-labels.";
   
   
   /** URI for NodeLabelManager */
   /** URI for NodeLabelManager */
   public static final String FS_NODE_LABELS_STORE_ROOT_DIR = NODE_LABELS_PREFIX
   public static final String FS_NODE_LABELS_STORE_ROOT_DIR = NODE_LABELS_PREFIX
@@ -1664,6 +1660,14 @@ public class YarnConfiguration extends Configuration {
       NODE_LABELS_PREFIX + "fs-store.retry-policy-spec";
       NODE_LABELS_PREFIX + "fs-store.retry-policy-spec";
   public static final String DEFAULT_FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC =
   public static final String DEFAULT_FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC =
       "2000, 500";
       "2000, 500";
+  
+  /**
+   * Flag to indicate if the node labels feature enabled, by default it's
+   * disabled
+   */
+  public static final String NODE_LABELS_ENABLED = NODE_LABELS_PREFIX
+      + "enabled";
+  public static final boolean DEFAULT_NODE_LABELS_ENABLED = false;
 
 
   public YarnConfiguration() {
   public YarnConfiguration() {
     super();
     super();

+ 1 - 0
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java

@@ -80,6 +80,7 @@ public class TestDistributedShell {
     conf.set("yarn.log.dir", "target");
     conf.set("yarn.log.dir", "target");
     conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
     conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
     conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
     conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
     
     
     if (yarnCluster == null) {
     if (yarnCluster == null) {
       yarnCluster =
       yarnCluster =

+ 41 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java

@@ -41,6 +41,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.event.EventHandler;
@@ -52,6 +53,7 @@ import org.apache.hadoop.yarn.nodelabels.event.RemoveClusterNodeLabels;
 import org.apache.hadoop.yarn.nodelabels.event.UpdateNodeToLabelsMappingsEvent;
 import org.apache.hadoop.yarn.nodelabels.event.UpdateNodeToLabelsMappingsEvent;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSet;
 
 
 public class CommonNodeLabelsManager extends AbstractService {
 public class CommonNodeLabelsManager extends AbstractService {
@@ -64,6 +66,14 @@ public class CommonNodeLabelsManager extends AbstractService {
   private static final Pattern LABEL_PATTERN = Pattern
   private static final Pattern LABEL_PATTERN = Pattern
       .compile("^[0-9a-zA-Z][0-9a-zA-Z-_]*");
       .compile("^[0-9a-zA-Z][0-9a-zA-Z-_]*");
   public static final int WILDCARD_PORT = 0;
   public static final int WILDCARD_PORT = 0;
+  
+  /**
+   * Error messages
+   */
+  @VisibleForTesting
+  public static final String NODE_LABELS_NOT_ENABLED_ERR =
+      "Node-label-based scheduling is disabled. Please check "
+          + YarnConfiguration.NODE_LABELS_ENABLED;
 
 
   /**
   /**
    * If a user doesn't specify label of a queue or node, it belongs
    * If a user doesn't specify label of a queue or node, it belongs
@@ -82,6 +92,7 @@ public class CommonNodeLabelsManager extends AbstractService {
   protected final WriteLock writeLock;
   protected final WriteLock writeLock;
 
 
   protected NodeLabelsStore store;
   protected NodeLabelsStore store;
+  private boolean nodeLabelsEnabled = false;
 
 
   /**
   /**
    * A <code>Host</code> can have multiple <code>Node</code>s 
    * A <code>Host</code> can have multiple <code>Node</code>s 
@@ -193,7 +204,13 @@ public class CommonNodeLabelsManager extends AbstractService {
 
 
   @Override
   @Override
   protected void serviceInit(Configuration conf) throws Exception {
   protected void serviceInit(Configuration conf) throws Exception {
-    initNodeLabelStore(conf);
+    // set if node labels enabled
+    nodeLabelsEnabled =
+        conf.getBoolean(YarnConfiguration.NODE_LABELS_ENABLED,
+            YarnConfiguration.DEFAULT_NODE_LABELS_ENABLED);
+    if (nodeLabelsEnabled) {
+      initNodeLabelStore(conf);
+    }
     
     
     labelCollections.put(NO_LABEL, new NodeLabel(NO_LABEL));
     labelCollections.put(NO_LABEL, new NodeLabel(NO_LABEL));
   }
   }
@@ -250,6 +267,10 @@ public class CommonNodeLabelsManager extends AbstractService {
    */
    */
   @SuppressWarnings("unchecked")
   @SuppressWarnings("unchecked")
   public void addToCluserNodeLabels(Set<String> labels) throws IOException {
   public void addToCluserNodeLabels(Set<String> labels) throws IOException {
+    if (!nodeLabelsEnabled) {
+      LOG.error(NODE_LABELS_NOT_ENABLED_ERR);
+      throw new IOException(NODE_LABELS_NOT_ENABLED_ERR);
+    }
     if (null == labels || labels.isEmpty()) {
     if (null == labels || labels.isEmpty()) {
       return;
       return;
     }
     }
@@ -304,6 +325,10 @@ public class CommonNodeLabelsManager extends AbstractService {
    */
    */
   public void addLabelsToNode(Map<NodeId, Set<String>> addedLabelsToNode)
   public void addLabelsToNode(Map<NodeId, Set<String>> addedLabelsToNode)
       throws IOException {
       throws IOException {
+    if (!nodeLabelsEnabled) {
+      LOG.error(NODE_LABELS_NOT_ENABLED_ERR);
+      throw new IOException(NODE_LABELS_NOT_ENABLED_ERR);
+    }
     addedLabelsToNode = normalizeNodeIdToLabels(addedLabelsToNode);
     addedLabelsToNode = normalizeNodeIdToLabels(addedLabelsToNode);
     checkAddLabelsToNode(addedLabelsToNode);
     checkAddLabelsToNode(addedLabelsToNode);
     internalUpdateLabelsOnNodes(addedLabelsToNode, NodeLabelUpdateOperation.ADD);
     internalUpdateLabelsOnNodes(addedLabelsToNode, NodeLabelUpdateOperation.ADD);
@@ -370,6 +395,11 @@ public class CommonNodeLabelsManager extends AbstractService {
    */
    */
   public void removeFromClusterNodeLabels(Collection<String> labelsToRemove)
   public void removeFromClusterNodeLabels(Collection<String> labelsToRemove)
       throws IOException {
       throws IOException {
+    if (!nodeLabelsEnabled) {
+      LOG.error(NODE_LABELS_NOT_ENABLED_ERR);
+      throw new IOException(NODE_LABELS_NOT_ENABLED_ERR);
+    }
+    
     labelsToRemove = normalizeLabels(labelsToRemove);
     labelsToRemove = normalizeLabels(labelsToRemove);
     
     
     checkRemoveFromClusterNodeLabels(labelsToRemove);
     checkRemoveFromClusterNodeLabels(labelsToRemove);
@@ -532,6 +562,11 @@ public class CommonNodeLabelsManager extends AbstractService {
   public void
   public void
       removeLabelsFromNode(Map<NodeId, Set<String>> removeLabelsFromNode)
       removeLabelsFromNode(Map<NodeId, Set<String>> removeLabelsFromNode)
           throws IOException {
           throws IOException {
+    if (!nodeLabelsEnabled) {
+      LOG.error(NODE_LABELS_NOT_ENABLED_ERR);
+      throw new IOException(NODE_LABELS_NOT_ENABLED_ERR);
+    }
+    
     removeLabelsFromNode = normalizeNodeIdToLabels(removeLabelsFromNode);
     removeLabelsFromNode = normalizeNodeIdToLabels(removeLabelsFromNode);
     
     
     checkRemoveLabelsFromNode(removeLabelsFromNode);
     checkRemoveLabelsFromNode(removeLabelsFromNode);
@@ -567,6 +602,11 @@ public class CommonNodeLabelsManager extends AbstractService {
    */
    */
   public void replaceLabelsOnNode(Map<NodeId, Set<String>> replaceLabelsToNode)
   public void replaceLabelsOnNode(Map<NodeId, Set<String>> replaceLabelsToNode)
       throws IOException {
       throws IOException {
+    if (!nodeLabelsEnabled) {
+      LOG.error(NODE_LABELS_NOT_ENABLED_ERR);
+      throw new IOException(NODE_LABELS_NOT_ENABLED_ERR);
+    }
+    
     replaceLabelsToNode = normalizeNodeIdToLabels(replaceLabelsToNode);
     replaceLabelsToNode = normalizeNodeIdToLabels(replaceLabelsToNode);
     
     
     checkReplaceLabelsOnNode(replaceLabelsToNode);
     checkReplaceLabelsOnNode(replaceLabelsToNode);

+ 60 - 2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java

@@ -26,6 +26,8 @@ import java.util.Set;
 
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.junit.After;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Before;
@@ -41,7 +43,9 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase {
   @Before
   @Before
   public void before() {
   public void before() {
     mgr = new DummyCommonNodeLabelsManager();
     mgr = new DummyCommonNodeLabelsManager();
-    mgr.init(new Configuration());
+    Configuration conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+    mgr.init(conf);
     mgr.start();
     mgr.start();
   }
   }
 
 
@@ -319,4 +323,58 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase {
         toSet("p1"), toNodeId("n1:1"), toSet("p2"), toNodeId("n1:2"),
         toSet("p1"), toNodeId("n1:1"), toSet("p2"), toNodeId("n1:2"),
         toSet("p1")));
         toSet("p1")));
   }
   }
-}
+
+  private void assertNodeLabelsDisabledErrorMessage(IOException e) {
+    Assert.assertEquals(CommonNodeLabelsManager.NODE_LABELS_NOT_ENABLED_ERR,
+        e.getMessage());
+  }
+  
+  @Test(timeout = 5000)
+  public void testNodeLabelsDisabled() throws IOException {
+    DummyCommonNodeLabelsManager mgr = new DummyCommonNodeLabelsManager();
+    Configuration conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+    mgr.init(conf);
+    mgr.start();
+    
+    // add labels
+    try {
+      mgr.addToCluserNodeLabels(ImmutableSet.of("x"));
+    } catch (IOException e) {
+      assertNodeLabelsDisabledErrorMessage(e);
+    }
+    
+    // remove labels
+    try {
+      mgr.removeFromClusterNodeLabels(ImmutableSet.of("x"));
+    } catch (IOException e) {
+      assertNodeLabelsDisabledErrorMessage(e);
+    }
+    
+    // add labels to node
+    try {
+      mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("host", 0),
+          CommonNodeLabelsManager.EMPTY_STRING_SET));
+    } catch (IOException e) {
+      assertNodeLabelsDisabledErrorMessage(e);
+    }
+    
+    // remove labels from node
+    try {
+      mgr.removeLabelsFromNode(ImmutableMap.of(NodeId.newInstance("host", 0),
+          CommonNodeLabelsManager.EMPTY_STRING_SET));
+    } catch (IOException e) {
+      assertNodeLabelsDisabledErrorMessage(e);
+    }
+    
+    // replace labels on node
+    try {
+      mgr.replaceLabelsOnNode(ImmutableMap.of(NodeId.newInstance("host", 0),
+          CommonNodeLabelsManager.EMPTY_STRING_SET));
+    } catch (IOException e) {
+      assertNodeLabelsDisabledErrorMessage(e);
+    }
+    
+    mgr.close();
+  }
+}

+ 1 - 0
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java

@@ -63,6 +63,7 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
   public void before() throws IOException {
   public void before() throws IOException {
     mgr = new MockNodeLabelManager();
     mgr = new MockNodeLabelManager();
     conf = new Configuration();
     conf = new Configuration();
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
     File tempDir = File.createTempFile("nlb", ".tmp");
     File tempDir = File.createTempFile("nlb", ".tmp");
     tempDir.delete();
     tempDir.delete();
     tempDir.mkdirs();
     tempDir.mkdirs();

+ 1 - 5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java

@@ -59,7 +59,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMaste
 import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
 import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
 import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy;
 import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy;
 import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor;
 import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
@@ -334,10 +333,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
   
   
   protected RMNodeLabelsManager createNodeLabelManager()
   protected RMNodeLabelsManager createNodeLabelManager()
       throws InstantiationException, IllegalAccessException {
       throws InstantiationException, IllegalAccessException {
-    Class<? extends RMNodeLabelsManager> nlmCls =
-        conf.getClass(YarnConfiguration.RM_NODE_LABELS_MANAGER_CLASS,
-            MemoryRMNodeLabelsManager.class, RMNodeLabelsManager.class);
-    return nlmCls.newInstance();
+    return new RMNodeLabelsManager();
   }
   }
   
   
   protected DelegationTokenRenewer createDelegationTokenRenewer() {
   protected DelegationTokenRenewer createDelegationTokenRenewer() {

+ 2 - 2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java

@@ -59,7 +59,7 @@ import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
 import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
 import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -115,7 +115,7 @@ public class MockRM extends ResourceManager {
   
   
   @Override
   @Override
   protected RMNodeLabelsManager createNodeLabelManager() {
   protected RMNodeLabelsManager createNodeLabelManager() {
-    RMNodeLabelsManager mgr = new MemoryRMNodeLabelsManager();
+    RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
     mgr.init(getConfig());
     mgr.init(getConfig());
     return mgr;
     return mgr;
   }
   }

+ 1 - 0
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java

@@ -2073,6 +2073,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
     
     
     MemoryRMStateStore memStore = new MemoryRMStateStore();
     MemoryRMStateStore memStore = new MemoryRMStateStore();
     memStore.init(conf);
     memStore.init(conf);
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
     MockRM rm1 = new MockRM(conf, memStore) {
     MockRM rm1 = new MockRM(conf, memStore) {
       @Override
       @Override
       protected RMNodeLabelsManager createNodeLabelManager() {
       protected RMNodeLabelsManager createNodeLabelManager() {

+ 9 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/MemoryRMNodeLabelsManager.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java

@@ -25,9 +25,10 @@ import java.util.Set;
 
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.nodelabels.NodeLabelsStore;
 import org.apache.hadoop.yarn.nodelabels.NodeLabelsStore;
 
 
-public class MemoryRMNodeLabelsManager extends RMNodeLabelsManager {
+public class NullRMNodeLabelsManager extends RMNodeLabelsManager {
   Map<NodeId, Set<String>> lastNodeToLabels = null;
   Map<NodeId, Set<String>> lastNodeToLabels = null;
   Collection<String> lastAddedlabels = null;
   Collection<String> lastAddedlabels = null;
   Collection<String> lastRemovedlabels = null;
   Collection<String> lastRemovedlabels = null;
@@ -79,4 +80,11 @@ public class MemoryRMNodeLabelsManager extends RMNodeLabelsManager {
   protected void stopDispatcher() {
   protected void stopDispatcher() {
     // do nothing
     // do nothing
   }
   }
+  
+  @Override
+  protected void serviceInit(Configuration conf) throws Exception {
+    // always enable node labels while using MemoryRMNodeLabelsManager
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+    super.serviceInit(conf);
+  }
 }
 }

+ 6 - 3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java

@@ -27,6 +27,7 @@ import java.util.Set;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
 import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
 import org.apache.hadoop.yarn.nodelabels.NodeLabel;
 import org.apache.hadoop.yarn.nodelabels.NodeLabel;
 import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase;
 import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase;
@@ -44,12 +45,14 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
   private final Resource SMALL_RESOURCE = Resource.newInstance(100, 0);
   private final Resource SMALL_RESOURCE = Resource.newInstance(100, 0);
   private final Resource LARGE_NODE = Resource.newInstance(1000, 0);
   private final Resource LARGE_NODE = Resource.newInstance(1000, 0);
   
   
-  MemoryRMNodeLabelsManager mgr = null;
+  NullRMNodeLabelsManager mgr = null;
 
 
   @Before
   @Before
   public void before() {
   public void before() {
-    mgr = new MemoryRMNodeLabelsManager();
-    mgr.init(new Configuration());
+    mgr = new NullRMNodeLabelsManager();
+    Configuration conf = new Configuration();
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+    mgr.init(conf);
     mgr.start();
     mgr.start();
   }
   }
 
 

+ 3 - 3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java

@@ -84,7 +84,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.Task;
 import org.apache.hadoop.yarn.server.resourcemanager.Task;
 import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS;
 import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS;
 import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
 import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
@@ -154,7 +154,7 @@ public class TestCapacityScheduler {
     resourceManager = new ResourceManager() {
     resourceManager = new ResourceManager() {
       @Override
       @Override
       protected RMNodeLabelsManager createNodeLabelManager() {
       protected RMNodeLabelsManager createNodeLabelManager() {
-        RMNodeLabelsManager mgr = new MemoryRMNodeLabelsManager();
+        RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
         mgr.init(getConfig());
         mgr.init(getConfig());
         return mgr;
         return mgr;
       }
       }
@@ -1485,7 +1485,7 @@ public class TestCapacityScheduler {
     resourceManager = new ResourceManager() {
     resourceManager = new ResourceManager() {
        @Override
        @Override
         protected RMNodeLabelsManager createNodeLabelManager() {
         protected RMNodeLabelsManager createNodeLabelManager() {
-          RMNodeLabelsManager mgr = new MemoryRMNodeLabelsManager();
+          RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
           mgr.init(getConfig());
           mgr.init(getConfig());
           return mgr;
           return mgr;
         }
         }

+ 2 - 2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java

@@ -29,7 +29,7 @@ import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
 import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
 import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
 import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
@@ -54,7 +54,7 @@ public class TestCapacitySchedulerNodeLabelUpdate {
     conf = new YarnConfiguration();
     conf = new YarnConfiguration();
     conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
     conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
       ResourceScheduler.class);
       ResourceScheduler.class);
-    mgr = new MemoryRMNodeLabelsManager();
+    mgr = new NullRMNodeLabelsManager();
     mgr.init(conf);
     mgr.init(conf);
   }
   }
   
   

+ 3 - 3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java

@@ -45,7 +45,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.RMSecretManagerService;
 import org.apache.hadoop.yarn.server.resourcemanager.RMSecretManagerService;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.TestFifoScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.TestFifoScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -81,7 +81,7 @@ public class TestContainerAllocation {
     conf = new YarnConfiguration();
     conf = new YarnConfiguration();
     conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
     conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
       ResourceScheduler.class);
       ResourceScheduler.class);
-    mgr = new MemoryRMNodeLabelsManager();
+    mgr = new NullRMNodeLabelsManager();
     mgr.init(conf);
     mgr.init(conf);
   }
   }
 
 
@@ -451,7 +451,7 @@ public class TestContainerAllocation {
   
   
   @Test(timeout = 300000)
   @Test(timeout = 300000)
   public void testContainerAllocationWithSingleUserLimits() throws Exception {
   public void testContainerAllocationWithSingleUserLimits() throws Exception {
-    final RMNodeLabelsManager mgr = new MemoryRMNodeLabelsManager();
+    final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
     mgr.init(conf);
     mgr.init(conf);
 
 
     // set node -> label
     // set node -> label

+ 6 - 6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java

@@ -26,7 +26,7 @@ import org.apache.hadoop.service.ServiceOperations;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
 import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
@@ -47,7 +47,7 @@ public class TestQueueParsing {
   
   
   @Before
   @Before
   public void setup() {
   public void setup() {
-    nodeLabelManager = new MemoryRMNodeLabelsManager();
+    nodeLabelManager = new NullRMNodeLabelsManager();
     nodeLabelManager.init(new YarnConfiguration());
     nodeLabelManager.init(new YarnConfiguration());
     nodeLabelManager.start();
     nodeLabelManager.start();
   }
   }
@@ -566,7 +566,7 @@ public class TestQueueParsing {
             new NMTokenSecretManagerInRM(csConf),
             new NMTokenSecretManagerInRM(csConf),
             new ClientToAMTokenSecretManagerInRM(), null);
             new ClientToAMTokenSecretManagerInRM(), null);
     
     
-    RMNodeLabelsManager nodeLabelsManager = new MemoryRMNodeLabelsManager();
+    RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
     nodeLabelsManager.init(conf);
     nodeLabelsManager.init(conf);
     nodeLabelsManager.start();
     nodeLabelsManager.start();
     
     
@@ -594,7 +594,7 @@ public class TestQueueParsing {
             new NMTokenSecretManagerInRM(csConf),
             new NMTokenSecretManagerInRM(csConf),
             new ClientToAMTokenSecretManagerInRM(), null);
             new ClientToAMTokenSecretManagerInRM(), null);
     
     
-    RMNodeLabelsManager nodeLabelsManager = new MemoryRMNodeLabelsManager();
+    RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
     nodeLabelsManager.init(conf);
     nodeLabelsManager.init(conf);
     nodeLabelsManager.start();
     nodeLabelsManager.start();
     
     
@@ -622,7 +622,7 @@ public class TestQueueParsing {
             new NMTokenSecretManagerInRM(csConf),
             new NMTokenSecretManagerInRM(csConf),
             new ClientToAMTokenSecretManagerInRM(), null);
             new ClientToAMTokenSecretManagerInRM(), null);
     
     
-    RMNodeLabelsManager nodeLabelsManager = new MemoryRMNodeLabelsManager();
+    RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
     nodeLabelsManager.init(conf);
     nodeLabelsManager.init(conf);
     nodeLabelsManager.start();
     nodeLabelsManager.start();
     
     
@@ -649,7 +649,7 @@ public class TestQueueParsing {
             new NMTokenSecretManagerInRM(csConf),
             new NMTokenSecretManagerInRM(csConf),
             new ClientToAMTokenSecretManagerInRM(), null);
             new ClientToAMTokenSecretManagerInRM(), null);
     
     
-    RMNodeLabelsManager nodeLabelsManager = new MemoryRMNodeLabelsManager();
+    RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
     nodeLabelsManager.init(conf);
     nodeLabelsManager.init(conf);
     nodeLabelsManager.start();
     nodeLabelsManager.start();
     
     

+ 3 - 3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java

@@ -39,7 +39,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.MockAsm;
 import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.MockAsm;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -179,7 +179,7 @@ public class TestRMWebApp {
          return nodesMap;
          return nodesMap;
        }
        }
      }; 
      }; 
-    rmContext.setNodeLabelManager(new MemoryRMNodeLabelsManager());
+    rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
     return rmContext;
     return rmContext;
   }
   }
 
 
@@ -211,7 +211,7 @@ public class TestRMWebApp {
         null, new RMContainerTokenSecretManager(conf),
         null, new RMContainerTokenSecretManager(conf),
         new NMTokenSecretManagerInRM(conf),
         new NMTokenSecretManagerInRM(conf),
         new ClientToAMTokenSecretManagerInRM(), null);
         new ClientToAMTokenSecretManagerInRM(), null);
-    rmContext.setNodeLabelManager(new MemoryRMNodeLabelsManager());
+    rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
     cs.setRMContext(rmContext);
     cs.setRMContext(rmContext);
     cs.init(conf);
     cs.init(conf);
     return cs;
     return cs;