YARN-2715. Fixed ResourceManager to respect common configurations for proxy users/groups beyond just the YARN level config. Contributed by Zhijie Shen.

(cherry picked from commit c0e034336c85296be6f549d88d137fb2b2b79a15)

hadoop-yarn-project/CHANGES.txt

@@ -555,6 +555,9 @@ Release 2.6.0 - UNRELEASED
     test-failures: added a dummy in-memory labels-manager. (Wangda Tan via
     vinodkv)
 
+    YARN-2715. Fixed ResourceManager to respect common configurations for proxy
+    users/groups beyond just the YARN level config. (Zhijie Shen via vinodkv)
+
   BREAKDOWN OF YARN-1051 SUBTASKS AND RELATED JIRAS
 
     YARN-1707. Introduce APIs to add/remove/resize queues in the

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -31,6 +31,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.authorize.ProxyUsers;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 
@@ -560,6 +561,11 @@ public class YarnConfiguration extends Configuration {
   public static final String DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION =
       "NONE";
 
+  /**
+   * RM proxy users' prefix
+   */
+  public static final String RM_PROXY_USER_PREFIX = RM_PREFIX + "proxyuser.";
+
   ////////////////////////////////
   // Node Manager Configs
   ////////////////////////////////

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java

@@ -35,6 +35,7 @@ import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
+import org.apache.hadoop.security.authorize.ProxyUsers;
 import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler;
 import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 
@@ -42,14 +43,12 @@ import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 public class RMAuthenticationFilterInitializer extends FilterInitializer {
 
   String configPrefix;
-  String proxyPrefix;
   String signatureSecretFileProperty;
   String kerberosPrincipalProperty;
   String cookiePath;
 
   public RMAuthenticationFilterInitializer() {
     this.configPrefix = "hadoop.http.authentication.";
-    this.proxyPrefix = "yarn.resourcemanager.webapp.proxyuser.";
     this.signatureSecretFileProperty =
         AuthenticationFilter.SIGNATURE_SECRET + ".file";
     this.kerberosPrincipalProperty = KerberosAuthenticationHandler.PRINCIPAL;
@@ -62,15 +61,18 @@ public class RMAuthenticationFilterInitializer extends FilterInitializer {
     // setting the cookie path to root '/' so it is used for all resources.
     filterConfig.put(AuthenticationFilter.COOKIE_PATH, cookiePath);
 
+    // Before conf object is passed in, RM has already processed it and used RM
+    // specific configs to overwrite hadoop common ones. Hence we just need to
+    // source hadoop.proxyuser configs here.
     for (Map.Entry<String, String> entry : conf) {
       String propName = entry.getKey();
       if (propName.startsWith(configPrefix)) {
         String value = conf.get(propName);
         String name = propName.substring(configPrefix.length());
         filterConfig.put(name, value);
-      } else if (propName.startsWith(proxyPrefix)) {
+      } else if (propName.startsWith(ProxyUsers.CONF_HADOOP_PROXYUSER)) {
         String value = conf.get(propName);
-        String name = propName.substring("yarn.resourcemanager.webapp.".length());
+        String name = propName.substring("hadoop.".length());
         filterConfig.put(name, value);
       }
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java

@@ -421,9 +421,13 @@ public class AdminService extends CompositeService implements
       throwStandbyException();
     }
 
+    // Accept hadoop common configs in core-site.xml as well as RM specific
+    // configurations in yarn-site.xml
     Configuration conf =
         getConfiguration(new Configuration(false),
-            YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
+            YarnConfiguration.CORE_SITE_CONFIGURATION_FILE,
+            YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
+    RMServerUtils.processRMProxyUsersConf(conf);
     ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
     RMAuditLogger.logSuccess(user.getShortUserName(),
         argName, "AdminService");
@@ -594,11 +598,13 @@ public class AdminService extends CompositeService implements
   }
 
   private synchronized Configuration getConfiguration(Configuration conf,
-      String confFileName) throws YarnException, IOException {
-    InputStream confFileInputStream = this.rmContext.getConfigurationProvider()
-        .getConfigurationInputStream(conf, confFileName);
-    if (confFileInputStream != null) {
-      conf.addResource(confFileInputStream);
+      String... confFileNames) throws YarnException, IOException {
+    for (String confFileName : confFileNames) {
+      InputStream confFileInputStream = this.rmContext.getConfigurationProvider()
+          .getConfigurationInputStream(conf, confFileName);
+      if (confFileInputStream != null) {
+        conf.addResource(confFileInputStream);
+      }
     }
     return conf;
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java

@@ -21,12 +21,16 @@ package org.apache.hadoop.yarn.server.resourcemanager;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.commons.logging.Log;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.security.authorize.ProxyUsers;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
 import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -36,6 +40,7 @@ import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
 import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
 import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
@@ -248,4 +253,26 @@ public class RMServerUtils {
       BuilderUtils.newApplicationResourceUsageReport(-1, -1,
           Resources.createResource(-1, -1), Resources.createResource(-1, -1),
           Resources.createResource(-1, -1), 0, 0);
+
+
+
+  /**
+   * Find all configs whose name starts with
+   * YarnConfiguration.RM_PROXY_USER_PREFIX, and add a record for each one by
+   * replacing the prefix with ProxyUsers.CONF_HADOOP_PROXYUSER
+   */
+  public static void processRMProxyUsersConf(Configuration conf) {
+    Map<String, String> rmProxyUsers = new HashMap<String, String>();
+    for (Map.Entry<String, String> entry : conf) {
+      String propName = entry.getKey();
+      if (propName.startsWith(YarnConfiguration.RM_PROXY_USER_PREFIX)) {
+        rmProxyUsers.put(ProxyUsers.CONF_HADOOP_PROXYUSER + "." +
+            propName.substring(YarnConfiguration.RM_PROXY_USER_PREFIX.length()),
+            entry.getValue());
+      }
+    }
+    for (Map.Entry<String, String> entry : rmProxyUsers.entrySet()) {
+      conf.set(entry.getKey(), entry.getValue());
+    }
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java

@@ -23,7 +23,9 @@ import java.io.InputStream;
 import java.net.InetSocketAddress;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 
@@ -212,6 +214,9 @@ public class ResourceManager extends CompositeService implements Recoverable {
         .refresh();
 
     // Do refreshSuperUserGroupsConfiguration with loaded core-site.xml
+    // Or use RM specific configurations to overwrite the common ones first
+    // if they exist
+    RMServerUtils.processRMProxyUsersConf(conf);
     ProxyUsers.refreshSuperUserGroupsConfiguration(this.conf);
 
     // load yarn-site.xml

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java

@@ -357,6 +357,24 @@ public class TestRMAdminService {
         .get("hadoop.proxyuser.test.hosts").size() == 1);
     Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts()
         .get("hadoop.proxyuser.test.hosts").contains("test_hosts"));
+
+    Configuration yarnConf = new Configuration(false);
+    yarnConf.set("yarn.resourcemanager.proxyuser.test.groups", "test_groups_1");
+    yarnConf.set("yarn.resourcemanager.proxyuser.test.hosts", "test_hosts_1");
+    uploadConfiguration(yarnConf, "yarn-site.xml");
+
+    // RM specific configs will overwrite the common ones
+    rm.adminService.refreshSuperUserGroupsConfiguration(
+        RefreshSuperUserGroupsConfigurationRequest.newInstance());
+    Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyGroups()
+        .get("hadoop.proxyuser.test.groups").size() == 1);
+    Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyGroups()
+        .get("hadoop.proxyuser.test.groups").contains("test_groups_1"));
+
+    Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts()
+        .get("hadoop.proxyuser.test.hosts").size() == 1);
+    Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts()
+        .get("hadoop.proxyuser.test.hosts").contains("test_hosts_1"));
   }
 
   @Test

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMProxyUsersConf.java

@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.security.authorize.ProxyUsers;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class TestRMProxyUsersConf {
+
+  private static final UserGroupInformation FOO_USER =
+      UserGroupInformation.createUserForTesting("foo", new String[] { "foo_group" });
+  private static final UserGroupInformation BAR_USER =
+      UserGroupInformation.createUserForTesting("bar", new String[] { "bar_group" });
+
+  @Parameterized.Parameters
+  public static Collection<Object[]> headers() {
+    return Arrays.asList(new Object[][] { { 0 }, { 1 }, { 2 } });
+  }
+
+  private Configuration conf;
+
+  public TestRMProxyUsersConf(int round) {
+    conf = new YarnConfiguration();
+    switch (round) {
+      case 0:
+        // hadoop.proxyuser prefix
+        conf.set("hadoop.proxyuser.foo.hosts", "localhost");
+        conf.set("hadoop.proxyuser.foo.users", "bar");
+        conf.set("hadoop.proxyuser.foo.groups", "bar_group");
+        break;
+      case 1:
+        // yarn.resourcemanager.proxyuser prefix
+        conf.set("yarn.resourcemanager.proxyuser.foo.hosts", "localhost");
+        conf.set("yarn.resourcemanager.proxyuser.foo.users", "bar");
+        conf.set("yarn.resourcemanager.proxyuser.foo.groups", "bar_group");
+        break;
+      case 2:
+        // hadoop.proxyuser prefix has been overwritten by
+        // yarn.resourcemanager.proxyuser prefix
+        conf.set("hadoop.proxyuser.foo.hosts", "xyz");
+        conf.set("hadoop.proxyuser.foo.users", "xyz");
+        conf.set("hadoop.proxyuser.foo.groups", "xyz");
+        conf.set("yarn.resourcemanager.proxyuser.foo.hosts", "localhost");
+        conf.set("yarn.resourcemanager.proxyuser.foo.users", "bar");
+        conf.set("yarn.resourcemanager.proxyuser.foo.groups", "bar_group");
+        break;
+      default:
+        break;
+    }
+  }
+
+  @Test
+  public void testProxyUserConfiguration() throws Exception {
+    MockRM rm = null;
+    try {
+      rm = new MockRM(conf);
+      rm.start();
+      // wait for web server starting
+      Thread.sleep(10000);
+      UserGroupInformation proxyUser =
+          UserGroupInformation.createProxyUser(
+              BAR_USER.getShortUserName(), FOO_USER);
+      try {
+        ProxyUsers.getDefaultImpersonationProvider().authorize(proxyUser,
+            "localhost");
+      } catch (AuthorizationException e) {
+        // Exception is not expected
+        Assert.fail();
+      }
+    } finally {
+      if (rm != null) {
+        rm.stop();
+        rm.close();
+      }
+    }
+  }
+
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java

@@ -158,8 +158,8 @@ public class TestRMWebServicesDelegationTokenAuthentication {
     rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
       httpSpnegoKeytabFile.getAbsolutePath());
     rmconf.setBoolean("mockrm.webapp.enabled", true);
-    rmconf.set("yarn.resourcemanager.webapp.proxyuser.client.hosts", "*");
-    rmconf.set("yarn.resourcemanager.webapp.proxyuser.client.groups", "*");
+    rmconf.set("yarn.resourcemanager.proxyuser.client.hosts", "*");
+    rmconf.set("yarn.resourcemanager.proxyuser.client.groups", "*");
     UserGroupInformation.setConfiguration(rmconf);
     rm = new MockRM(rmconf);
     rm.start();