YARN-1766. Fixed a bug in ResourceManager to use configuration loaded from the configuration-provider when booting up. Contributed by Xuan Gong.

svn merge --ignore-ancestry -c 1574252 ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1574253 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -376,6 +376,9 @@ Release 2.4.0 - UNRELEASED
     YARN-1729. Made TimelineWebServices deserialize the string primary- and
     secondary-filters param into the JSON-compatible object. (Billie Rinaldi via
     zjshen)
+
+    YARN-1766. Fixed a bug in ResourceManager to use configuration loaded from the
+    configuration-provider when booting up. (Xuan Gong via vinodkv)
     
 
 Release 2.3.1 - UNRELEASED

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java

@@ -36,6 +36,7 @@ import org.apache.hadoop.metrics2.source.JvmMetrics;
 import org.apache.hadoop.security.Groups;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.ProxyUsers;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.service.CompositeService;
 import org.apache.hadoop.service.Service;
@@ -181,7 +182,6 @@ public class ResourceManager extends CompositeService implements Recoverable {
 
   @Override
   protected void serviceInit(Configuration conf) throws Exception {
-    validateConfigs(conf);
     this.conf = conf;
     this.rmContext = new RMContextImpl();
 
@@ -190,13 +190,6 @@ public class ResourceManager extends CompositeService implements Recoverable {
     this.configurationProvider.init(this.conf);
     rmContext.setConfigurationProvider(configurationProvider);
 
-    // load yarn-site.xml
-    InputStream yarnSiteXMLInputStream =
-        this.configurationProvider.getConfigurationInputStream(this.conf,
-            YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
-    if (yarnSiteXMLInputStream != null) {
-      this.conf.addResource(yarnSiteXMLInputStream);
-    }
     // load core-site.xml
     InputStream coreSiteXMLInputStream =
         this.configurationProvider.getConfigurationInputStream(this.conf,
@@ -209,6 +202,19 @@ public class ResourceManager extends CompositeService implements Recoverable {
     Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf)
         .refresh();
 
+    // Do refreshSuperUserGroupsConfiguration with loaded core-site.xml
+    ProxyUsers.refreshSuperUserGroupsConfiguration(this.conf);
+
+    // load yarn-site.xml
+    InputStream yarnSiteXMLInputStream =
+        this.configurationProvider.getConfigurationInputStream(this.conf,
+            YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
+    if (yarnSiteXMLInputStream != null) {
+      this.conf.addResource(yarnSiteXMLInputStream);
+    }
+
+    validateConfigs(this.conf);
+
     // register the handlers for all AlwaysOn services using setupDispatcher().
     rmDispatcher = setupDispatcher();
     addIfService(rmDispatcher);
@@ -218,15 +224,15 @@ public class ResourceManager extends CompositeService implements Recoverable {
     addService(adminService);
     rmContext.setRMAdminService(adminService);
 
-    this.rmContext.setHAEnabled(HAUtil.isHAEnabled(conf));
+    this.rmContext.setHAEnabled(HAUtil.isHAEnabled(this.conf));
     if (this.rmContext.isHAEnabled()) {
-      HAUtil.verifyAndSetConfiguration(conf);
+      HAUtil.verifyAndSetConfiguration(this.conf);
     }
     createAndInitActiveServices();
 
-    webAppAddress = WebAppUtils.getRMWebAppURLWithoutScheme(conf);
+    webAppAddress = WebAppUtils.getRMWebAppURLWithoutScheme(this.conf);
 
-    super.serviceInit(conf);
+    super.serviceInit(this.conf);
   }
   
   protected QueueACLsManager createQueueACLsManager(ResourceScheduler scheduler,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java

@@ -590,6 +590,140 @@ public class TestRMAdminService {
 
   }
 
+  @Test
+  public void testRMInitialsWithFileSystemBasedConfigurationProvider()
+      throws Exception {
+    configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+        "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
+
+    // upload configurations
+    final File excludeHostsFile = new File(tmpDir.toString(), "excludeHosts");
+    if (excludeHostsFile.exists()) {
+      excludeHostsFile.delete();
+    }
+    if (!excludeHostsFile.createNewFile()) {
+      Assert.fail("Can not create " + "excludeHosts");
+    }
+    PrintWriter fileWriter = new PrintWriter(excludeHostsFile);
+    fileWriter.write("0.0.0.0:123");
+    fileWriter.close();
+    uploadToRemoteFileSystem(new Path(excludeHostsFile.getAbsolutePath()));
+
+    YarnConfiguration yarnConf = new YarnConfiguration();
+    yarnConf.set(YarnConfiguration.YARN_ADMIN_ACL, "world:anyone:rwcda");
+    yarnConf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, this.workingPath
+        + "/excludeHosts");
+    uploadConfiguration(yarnConf, "yarn-site.xml");
+
+    CapacitySchedulerConfiguration csConf =
+        new CapacitySchedulerConfiguration();
+    csConf.set("yarn.scheduler.capacity.maximum-applications", "5000");
+    uploadConfiguration(csConf, "capacity-scheduler.xml");
+
+    String aclsString = "alice,bob users,wheel";
+    Configuration newConf = new Configuration();
+    newConf.set("security.applicationclient.protocol.acl", aclsString);
+    uploadConfiguration(newConf, "hadoop-policy.xml");
+
+    Configuration conf = new Configuration();
+    conf.setBoolean(
+        CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true);
+    conf.set("hadoop.proxyuser.test.groups", "test_groups");
+    conf.set("hadoop.proxyuser.test.hosts", "test_hosts");
+    conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+        MockUnixGroupsMapping.class,
+        GroupMappingServiceProvider.class);
+    uploadConfiguration(conf, "core-site.xml");
+
+    // update the groups
+    MockUnixGroupsMapping.updateGroups();
+
+    ResourceManager resourceManager = null;
+    try {
+      try {
+        resourceManager = new ResourceManager();
+        resourceManager.init(configuration);
+        resourceManager.start();
+      } catch (Exception ex) {
+        fail("Should not get any exceptions");
+      }
+
+      // validate values for excludeHosts
+      Set<String> excludeHosts =
+          resourceManager.getRMContext().getNodesListManager()
+              .getHostsReader().getExcludedHosts();
+      Assert.assertTrue(excludeHosts.size() == 1);
+      Assert.assertTrue(excludeHosts.contains("0.0.0.0:123"));
+
+      // validate values for admin-acls
+      String aclStringAfter =
+          resourceManager.adminService.getAccessControlList()
+              .getAclString().trim();
+      Assert.assertEquals(aclStringAfter, "world:anyone:rwcda");
+
+      // validate values for queue configuration
+      CapacityScheduler cs =
+          (CapacityScheduler) resourceManager.getRMContext().getScheduler();
+      int maxAppsAfter = cs.getConfiguration().getMaximumSystemApplications();
+      Assert.assertEquals(maxAppsAfter, 5000);
+
+      // verify service Acls for AdminService
+      ServiceAuthorizationManager adminServiceServiceManager =
+          resourceManager.adminService.getServer()
+              .getServiceAuthorizationManager();
+      verifyServiceACLsRefresh(adminServiceServiceManager,
+          org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class,
+          aclsString);
+
+      // verify service ACLs for ClientRMService
+      ServiceAuthorizationManager clientRMServiceServiceManager =
+          resourceManager.getRMContext().getClientRMService().getServer()
+              .getServiceAuthorizationManager();
+      verifyServiceACLsRefresh(clientRMServiceServiceManager,
+          org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class,
+          aclsString);
+
+      // verify service ACLs for ApplicationMasterService
+      ServiceAuthorizationManager appMasterService =
+          resourceManager.getRMContext().getApplicationMasterService()
+              .getServer().getServiceAuthorizationManager();
+      verifyServiceACLsRefresh(appMasterService,
+          org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class,
+          aclsString);
+
+      // verify service ACLs for ResourceTrackerService
+      ServiceAuthorizationManager RTService =
+          resourceManager.getRMContext().getResourceTrackerService()
+              .getServer().getServiceAuthorizationManager();
+      verifyServiceACLsRefresh(RTService,
+          org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class,
+          aclsString);
+
+      // verify ProxyUsers and ProxyHosts
+      Assert.assertTrue(ProxyUsers.getProxyGroups()
+          .get("hadoop.proxyuser.test.groups").size() == 1);
+      Assert.assertTrue(ProxyUsers.getProxyGroups()
+          .get("hadoop.proxyuser.test.groups").contains("test_groups"));
+
+      Assert.assertTrue(ProxyUsers.getProxyHosts()
+          .get("hadoop.proxyuser.test.hosts").size() == 1);
+      Assert.assertTrue(ProxyUsers.getProxyHosts()
+          .get("hadoop.proxyuser.test.hosts").contains("test_hosts"));
+
+      // verify UserToGroupsMappings
+      List<String> groupAfter =
+          Groups.getUserToGroupsMappingService(configuration).getGroups(
+              UserGroupInformation.getCurrentUser().getUserName());
+      Assert.assertTrue(groupAfter.contains("test_group_D")
+          && groupAfter.contains("test_group_E")
+          && groupAfter.contains("test_group_F") && groupAfter.size() == 3);
+    } finally {
+      if (resourceManager != null) {
+        resourceManager.stop();
+      }
+    }
+  }
+
   private String writeConfigurationXML(Configuration conf, String confXMLName)
       throws IOException {
     DataOutputStream output = null;