YARN-1867. Fixed a bug in ResourceManager that was causing invalid ACL checks in the web-services after fail-over. Contributed by Vinod Kumar Vavilapalli.

svn merge --ignore-ancestry -c 1581662 ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2.4@1581665 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java

@@ -81,6 +81,7 @@ public class RMContextImpl implements RMContext {
   private ApplicationMasterService applicationMasterService;
   private RMApplicationHistoryWriter rmApplicationHistoryWriter;
   private ConfigurationProvider configurationProvider;
+
   /**
    * Default constructor. To be used in conjunction with setter methods for
    * individual fields.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java

@@ -141,19 +141,14 @@ public class ResourceManager extends CompositeService implements Recoverable {
   protected ResourceScheduler scheduler;
   private ClientRMService clientRM;
   protected ApplicationMasterService masterService;
-  private ApplicationMasterLauncher applicationMasterLauncher;
-  private ContainerAllocationExpirer containerAllocationExpirer;
   protected NMLivelinessMonitor nmLivelinessMonitor;
   protected NodesListManager nodesListManager;
-  private EventHandler<SchedulerEvent> schedulerDispatcher;
   protected RMAppManager rmAppManager;
   protected ApplicationACLsManager applicationACLsManager;
   protected QueueACLsManager queueACLsManager;
-  private DelegationTokenRenewer delegationTokenRenewer;
   private WebApp webApp;
   private AppReportFetcher fetcher = null;
   protected ResourceTrackerService resourceTracker;
-  private boolean recoveryEnabled;
 
   private String webAppAddress;
   private ConfigurationProvider configurationProvider = null;
@@ -333,6 +328,14 @@ public class ResourceManager extends CompositeService implements Recoverable {
    */
   @Private
   class RMActiveServices extends CompositeService {
+
+    private DelegationTokenRenewer delegationTokenRenewer;
+    private EventHandler<SchedulerEvent> schedulerDispatcher;
+    private ApplicationMasterLauncher applicationMasterLauncher;
+    private ContainerAllocationExpirer containerAllocationExpirer;
+
+    private boolean recoveryEnabled;
+
     RMActiveServices() {
       super("RMActiveServices");
     }
@@ -1009,6 +1012,11 @@ public class ResourceManager extends CompositeService implements Recoverable {
     return this.queueACLsManager;
   }
 
+  @Private
+  WebApp getWebapp() {
+    return this.webApp;
+  }
+
   @Override
   public void recover(RMState state) throws Exception {
     // recover RMdelegationTokenSecretManager
@@ -1055,16 +1063,14 @@ public class ResourceManager extends CompositeService implements Recoverable {
     rmContext.setDispatcher(rmDispatcher);
   }
 
-
   /**
    * Retrieve RM bind address from configuration
-   *
+   * 
    * @param conf
    * @return InetSocketAddress
    */
-public static InetSocketAddress getBindAddress(Configuration conf) {
+  public static InetSocketAddress getBindAddress(Configuration conf) {
     return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
-      YarnConfiguration.DEFAULT_RM_ADDRESS,
-      YarnConfiguration.DEFAULT_RM_PORT);
+      YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT);
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java

@@ -101,19 +101,12 @@ public class RMWebServices {
   private final ResourceManager rm;
   private static RecordFactory recordFactory = RecordFactoryProvider
       .getRecordFactory(null);
-  private final ApplicationACLsManager aclsManager;
-  private final QueueACLsManager queueACLsManager;
   private final Configuration conf;
   private @Context HttpServletResponse response;
 
   @Inject
-  public RMWebServices(final ResourceManager rm,
-      final ApplicationACLsManager aclsManager,
-      final QueueACLsManager queueACLsManager,
-      Configuration conf) {
+  public RMWebServices(final ResourceManager rm, Configuration conf) {
     this.rm = rm;
-    this.aclsManager = aclsManager;
-    this.queueACLsManager = queueACLsManager;
     this.conf = conf;
   }
 
@@ -125,10 +118,11 @@ public class RMWebServices {
       callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
     }
     if (callerUGI != null
-        && !(this.aclsManager.checkAccess(callerUGI,
-            ApplicationAccessType.VIEW_APP, app.getUser(),
-            app.getApplicationId()) || this.queueACLsManager.checkAccess(
-            callerUGI, QueueACL.ADMINISTER_QUEUE, app.getQueue()))) {
+        && !(this.rm.getApplicationACLsManager().checkAccess(callerUGI,
+              ApplicationAccessType.VIEW_APP, app.getUser(),
+              app.getApplicationId()) ||
+            this.rm.getQueueACLsManager().checkAccess(callerUGI,
+              QueueACL.ADMINISTER_QUEUE, app.getQueue()))) {
       return false;
     }
     return true;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java

@@ -80,6 +80,8 @@ import org.apache.log4j.Logger;
 @SuppressWarnings("unchecked")
 public class MockRM extends ResourceManager {
 
+  static final String ENABLE_WEBAPP = "mockrm.webapp.enabled";
+
   public MockRM() {
     this(new YarnConfiguration());
   }
@@ -494,7 +496,12 @@ public class MockRM extends ResourceManager {
 
   @Override
   protected void startWepApp() {
-    // override to disable webapp
+    if (getConfig().getBoolean(ENABLE_WEBAPP, false)) {
+      super.startWepApp();
+      return;
+    }
+
+    // Disable webapp
   }
 
   public static void finishAMAndVerifyAppState(RMApp rmApp, MockRM rm, MockNM nm,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java

@@ -18,6 +18,18 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import javax.ws.rs.core.MediaType;
+
+import junit.framework.Assert;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -25,10 +37,11 @@ import org.apache.hadoop.ha.HAServiceProtocol;
 import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
 import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
 import org.apache.hadoop.ha.HealthCheckFailedException;
+import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -36,17 +49,15 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
 import org.junit.Before;
 import org.junit.Test;
 
-import java.io.IOException;
-
-import junit.framework.Assert;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
 
 public class TestRMHA {
   private Log LOG = LogFactory.getLog(TestRMHA.class);
@@ -77,6 +88,10 @@ public class TestRMHA {
       configuration.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
       configuration.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS);
     }
+
+    // Enable webapp to test web-services also
+    configuration.setBoolean(MockRM.ENABLE_WEBAPP, true);
+    configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
   }
 
   private void checkMonitorHealth() throws IOException {
@@ -97,7 +112,7 @@ public class TestRMHA {
         rm.adminService.getServiceStatus().isReadyToBecomeActive());
   }
 
-  private void checkActiveRMFunctionality() throws IOException {
+  private void checkActiveRMFunctionality() throws Exception {
     assertEquals(STATE_ERR, HAServiceState.ACTIVE,
         rm.adminService.getServiceStatus().getState());
     assertTrue("Active RM services aren't started",
@@ -115,6 +130,33 @@ public class TestRMHA {
       fail("Unable to perform Active RM functions");
       LOG.error("ActiveRM check failed", e);
     }
+
+    checkActiveRMWebServices();
+  }
+
+  // Do some sanity testing of the web-services after fail-over.
+  private void checkActiveRMWebServices() throws JSONException {
+
+    // Validate web-service
+    Client webServiceClient = Client.create(new DefaultClientConfig());
+    InetSocketAddress rmWebappAddr =
+        NetUtils.getConnectAddress(rm.getWebapp().getListenerAddress());
+    String webappURL =
+        "http://" + rmWebappAddr.getHostName() + ":" + rmWebappAddr.getPort();
+    WebResource webResource = webServiceClient.resource(webappURL);
+    String path = app.getApplicationId().toString();
+
+    ClientResponse response =
+        webResource.path("ws").path("v1").path("cluster").path("apps")
+          .path(path).accept(MediaType.APPLICATION_JSON)
+          .get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    JSONObject json = response.getEntity(JSONObject.class);
+
+    assertEquals("incorrect number of elements", 1, json.length());
+    JSONObject appJson = json.getJSONObject("app");
+    assertEquals("ACCEPTED", appJson.getString("state"));
+    // Other stuff is verified in the regular web-services related tests
   }
 
   /**
@@ -129,9 +171,10 @@ public class TestRMHA {
    * become Active
    */
   @Test (timeout = 30000)
-  public void testStartAndTransitions() throws Exception {
+  public void testFailoverAndTransitions() throws Exception {
     configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
     Configuration conf = new YarnConfiguration(configuration);
+
     rm = new MockRM(conf);
     rm.init(conf);
     StateChangeRequestInfo requestInfo = new StateChangeRequestInfo(
@@ -191,7 +234,7 @@ public class TestRMHA {
   }
 
   @Test
-  public void testTransitionsWhenAutomaticFailoverEnabled() throws IOException {
+  public void testTransitionsWhenAutomaticFailoverEnabled() throws Exception {
     final String ERR_UNFORCED_REQUEST = "User request succeeded even when " +
         "automatic failover is enabled";