YARN-8319. More YARN pages need to honor yarn.resourcemanager.display.per-user-apps. Contributed by Sunil G.

(cherry picked from commit c05b5d424b000bab766f57e88a07f2b4e9a56647)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -121,6 +121,10 @@ public class YarnConfiguration extends Configuration {
         new DeprecationDelta(RM_ZK_RETRY_INTERVAL_MS,
             CommonConfigurationKeys.ZK_RETRY_INTERVAL_MS),
     });
+    Configuration.addDeprecations(new DeprecationDelta[] {
+        new DeprecationDelta("yarn.resourcemanager.display.per-user-apps",
+            FILTER_ENTITY_LIST_BY_USER)
+    });
   }
 
   //Configurations
@@ -3523,11 +3527,16 @@ public class YarnConfiguration extends Configuration {
   public static final String NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_SCRIPT_OPTS =
       NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PREFIX + "opts";
 
-  /*
+  /**
    * Support to view apps for given user in secure cluster.
+   * @deprecated This field is deprecated for {@link #FILTER_ENTITY_LIST_BY_USER}
    */
+  @Deprecated
   public static final String DISPLAY_APPS_FOR_LOGGED_IN_USER =
       RM_PREFIX + "display.per-user-apps";
+
+  public static final String FILTER_ENTITY_LIST_BY_USER =
+      "yarn.webapp.filter-entity-list-by-user";
   public static final boolean DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER =
       false;
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java

@@ -182,6 +182,8 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
     // Ignore deprecated properties
     configurationPrefixToSkipCompare
         .add(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS);
+    configurationPrefixToSkipCompare
+        .add(YarnConfiguration.DISPLAY_APPS_FOR_LOGGED_IN_USER);
 
     // Allocate for usage
     xmlPropsToSkipCompare = new HashSet<String>();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -3513,7 +3513,7 @@
   </property>
  
   <property>
-    <name>yarn.resourcemanager.display.per-user-apps</name>
+    <name>yarn.webapp.filter-entity-list-by-user</name>
     <value>false</value>
       <description>
         Flag to enable display of applications per user as an admin

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java

@@ -22,6 +22,7 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.charset.Charset;
+import java.security.Principal;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -54,6 +55,8 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.http.JettyUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -99,6 +102,7 @@ public class NMWebServices {
       .getRecordFactory(null);
   private final String redirectWSUrl;
   private final LogAggregationFileControllerFactory factory;
+  private boolean filterAppsByUser = false;
 
   private @javax.ws.rs.core.Context 
     HttpServletRequest request;
@@ -119,6 +123,15 @@ public class NMWebServices {
         YarnConfiguration.YARN_LOG_SERVER_WEBSERVICE_URL);
     this.factory = new LogAggregationFileControllerFactory(
         this.nmContext.getConf());
+    this.filterAppsByUser = this.nmContext.getConf().getBoolean(
+        YarnConfiguration.FILTER_ENTITY_LIST_BY_USER,
+        YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
+  }
+
+  public NMWebServices(final Context nm, final ResourceView view,
+      final WebApp webapp, HttpServletResponse response) {
+    this(nm, view, webapp);
+    this.response = response;
   }
 
   private void init() {
@@ -146,7 +159,8 @@ public class NMWebServices {
   @Path("/apps")
   @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
       MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
-  public AppsInfo getNodeApps(@QueryParam("state") String stateQuery,
+  public AppsInfo getNodeApps(@javax.ws.rs.core.Context HttpServletRequest hsr,
+      @QueryParam("state") String stateQuery,
       @QueryParam("user") String userQuery) {
     init();
     AppsInfo allApps = new AppsInfo();
@@ -169,6 +183,14 @@ public class NMWebServices {
           continue;
         }
       }
+
+      // Allow only application-owner/admin for any type of access on the
+      // application.
+      if (filterAppsByUser
+          && !hasAccess(appInfo.getUser(), entry.getKey(), hsr)) {
+        continue;
+      }
+
       allApps.add(appInfo);
     }
     return allApps;
@@ -205,6 +227,16 @@ public class NMWebServices {
       }
       ContainerInfo info = new ContainerInfo(this.nmContext, entry.getValue(),
           uriInfo.getBaseUri().toString(), webapp.name(), hsr.getRemoteUser());
+
+      ApplicationId appId = entry.getKey().getApplicationAttemptId()
+          .getApplicationId();
+      // Allow only application-owner/admin for any type of access on the
+      // application.
+      if (filterAppsByUser
+          && !hasAccess(entry.getValue().getUser(), appId, hsr)) {
+        continue;
+      }
+
       allContainers.add(info);
     }
     return allContainers;
@@ -553,4 +585,33 @@ public class NMWebServices {
     res.header("Location", redirectPath.toString());
     return res.build();
   }
+
+  protected Boolean hasAccess(String user, ApplicationId appId,
+      HttpServletRequest hsr) {
+    // Check for the authorization.
+    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+
+    if (callerUGI != null && !(this.nmContext.getApplicationACLsManager()
+        .checkAccess(callerUGI, ApplicationAccessType.VIEW_APP, user, appId))) {
+      return false;
+    }
+    return true;
+  }
+
+  private UserGroupInformation getCallerUserGroupInformation(
+      HttpServletRequest hsr, boolean usePrincipal) {
+
+    String remoteUser = hsr.getRemoteUser();
+    if (usePrincipal) {
+      Principal princ = hsr.getUserPrincipal();
+      remoteUser = princ == null ? null : princ.getName();
+    }
+
+    UserGroupInformation callerUGI = null;
+    if (remoteUser != null) {
+      callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
+    }
+
+    return callerUGI;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java

@@ -22,12 +22,17 @@ import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseS
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 import java.io.File;
 import java.io.IOException;
 import java.io.StringReader;
+import java.security.Principal;
 import java.util.HashMap;
 
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 import javax.ws.rs.core.MediaType;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
@@ -44,11 +49,13 @@ import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
 import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
 import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
 import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AppsInfo;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
@@ -192,25 +199,28 @@ public class TestNMWebServicesApps extends JerseyTestBase {
 
   private HashMap<String, String> addAppContainers(Application app) 
       throws IOException {
+    return addAppContainers(app, nmContext);
+  }
+
+  private HashMap<String, String> addAppContainers(Application app,
+      Context context) throws IOException {
     Dispatcher dispatcher = new AsyncDispatcher();
-    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
-        app.getAppId(), 1);
+    ApplicationAttemptId appAttemptId = BuilderUtils
+        .newApplicationAttemptId(app.getAppId(), 1);
     Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
         app.getUser(), app.getAppId(), 1);
     Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
         app.getUser(), app.getAppId(), 2);
-    nmContext.getContainers()
-        .put(container1.getContainerId(), container1);
-    nmContext.getContainers()
-        .put(container2.getContainerId(), container2);
+    context.getContainers().put(container1.getContainerId(), container1);
+    context.getContainers().put(container2.getContainerId(), container2);
 
     app.getContainers().put(container1.getContainerId(), container1);
     app.getContainers().put(container2.getContainerId(), container2);
     HashMap<String, String> hash = new HashMap<String, String>();
-    hash.put(container1.getContainerId().toString(), container1
-        .getContainerId().toString());
-    hash.put(container2.getContainerId().toString(), container2
-        .getContainerId().toString());
+    hash.put(container1.getContainerId().toString(),
+        container1.getContainerId().toString());
+    hash.put(container2.getContainerId().toString(),
+        container2.getContainerId().toString());
     return hash;
   }
 
@@ -721,4 +731,42 @@ public class TestNMWebServicesApps extends JerseyTestBase {
         user);
   }
 
+  @Test
+  public void testNodeAppsUserFiltering() throws JSONException, Exception {
+    Configuration yarnConf = new Configuration();
+    yarnConf.setBoolean(YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, true);
+    yarnConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+    yarnConf.setStrings(YarnConfiguration.YARN_ADMIN_ACL, "admin");
+    ApplicationACLsManager aclManager = new ApplicationACLsManager(yarnConf);
+
+    NMContext context = new NodeManager.NMContext(null, null, dirsHandler,
+        aclManager, null, false, yarnConf);
+    Application app = new MockApp(1);
+    context.getApplications().put(app.getAppId(), app);
+    addAppContainers(app, context);
+    Application app2 = new MockApp("foo", 1234, 2);
+    context.getApplications().put(app2.getAppId(), app2);
+    addAppContainers(app2, context);
+
+    // User "foo" could only see its own apps/containers.
+    NMWebServices webSvc = new NMWebServices(context, null, nmWebApp,
+        mock(HttpServletResponse.class));
+    HttpServletRequest mockHsr = mockHttpServletRequestByUserName("foo");
+    AppsInfo appsInfo = webSvc.getNodeApps(mockHsr, null, null);
+    assertEquals(1, appsInfo.getApps().size());
+
+    // Admin could see all apps and containers.
+    HttpServletRequest mockHsrAdmin = mockHttpServletRequestByUserName("admin");
+    AppsInfo appsInfo2 = webSvc.getNodeApps(mockHsrAdmin, null, null);
+    assertEquals(2, appsInfo2.getApps().size());
+  }
+
+  private HttpServletRequest mockHttpServletRequestByUserName(String username) {
+    HttpServletRequest mockHsr = mock(HttpServletRequest.class);
+    when(mockHsr.getRemoteUser()).thenReturn(username);
+    Principal principal = mock(Principal.class);
+    when(principal.getName()).thenReturn(username);
+    when(mockHsr.getUserPrincipal()).thenReturn(principal);
+    return mockHsr;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java

@@ -216,7 +216,7 @@ public class ClientRMService extends AbstractService implements
   private ReservationSystem reservationSystem;
   private ReservationInputValidator rValidator;
 
-  private boolean displayPerUserApps = false;
+  private boolean filterAppsByUser = false;
 
   private static final EnumSet<RMAppState> ACTIVE_APP_STATES = EnumSet.of(
       RMAppState.ACCEPTED, RMAppState.RUNNING);
@@ -283,8 +283,8 @@ public class ClientRMService extends AbstractService implements
       refreshServiceAcls(conf, RMPolicyProvider.getInstance());
     }
 
-    this.displayPerUserApps  = conf.getBoolean(
-        YarnConfiguration.DISPLAY_APPS_FOR_LOGGED_IN_USER,
+    this.filterAppsByUser  = conf.getBoolean(
+        YarnConfiguration.FILTER_ENTITY_LIST_BY_USER,
         YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
 
     this.server.start();
@@ -922,7 +922,7 @@ public class ClientRMService extends AbstractService implements
 
       // Given RM is configured to display apps per user, skip apps to which
       // this caller doesn't have access to view.
-      if (displayPerUserApps && !allowAccess) {
+      if (filterAppsByUser && !allowAccess) {
         continue;
       }
 
@@ -1840,6 +1840,6 @@ public class ClientRMService extends AbstractService implements
 
   @VisibleForTesting
   public void setDisplayPerUserApps(boolean displayPerUserApps) {
-    this.displayPerUserApps = displayPerUserApps;
+    this.filterAppsByUser = displayPerUserApps;
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java

@@ -228,7 +228,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
 
   @VisibleForTesting
   boolean isCentralizedNodeLabelConfiguration = true;
-  private boolean displayPerUserApps = false;
+  private boolean filterAppsByUser = false;
 
   public final static String DELEGATION_TOKEN_HEADER =
       "Hadoop-YARN-RM-Delegation-Token";
@@ -241,8 +241,8 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
     this.conf = conf;
     isCentralizedNodeLabelConfiguration =
         YarnConfiguration.isCentralizedNodeLabelConfiguration(conf);
-    this.displayPerUserApps  = conf.getBoolean(
-        YarnConfiguration.DISPLAY_APPS_FOR_LOGGED_IN_USER,
+    this.filterAppsByUser  = conf.getBoolean(
+        YarnConfiguration.FILTER_ENTITY_LIST_BY_USER,
         YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
   }
 
@@ -654,7 +654,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
       boolean allowAccess = hasAccess(rmapp, hsr);
       // Given RM is configured to display apps per user, skip apps to which
       // this caller doesn't have access to view.
-      if (displayPerUserApps && !allowAccess) {
+      if (filterAppsByUser && !allowAccess) {
         continue;
       }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java

@@ -23,6 +23,7 @@ import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Collections;
 import java.util.Date;
+import java.util.LinkedHashSet;
 import java.util.Locale;
 import java.util.Set;
 import java.util.TimeZone;
@@ -42,12 +43,15 @@ import javax.ws.rs.core.Response;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.JettyUtils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 import org.apache.hadoop.yarn.webapp.BadRequestException;
@@ -1450,6 +1454,19 @@ public class TimelineReaderWebServices {
     long endTime = Time.monotonicNow();
     if (entities == null) {
       entities = Collections.emptySet();
+    } else if (isDisplayEntityPerUserFilterEnabled(
+        timelineReaderManager.getConfig())) {
+      Set<TimelineEntity> userEntities = new LinkedHashSet<>();
+      userEntities.addAll(entities);
+      for (TimelineEntity entity : userEntities) {
+        if (entity.getInfo() != null) {
+          String userId =
+              (String) entity.getInfo().get(FlowActivityEntity.USER_INFO_KEY);
+          if (!validateAuthUserWithEntityUser(callerUGI, userId)) {
+            entities.remove(entity);
+          }
+        }
+      }
     }
     LOG.info("Processed URL " + url +
         " (Took " + (endTime - startTime) + " ms.)");
@@ -3403,4 +3420,20 @@ public class TimelineReaderWebServices {
         "Processed URL " + url + " (Took " + (endTime - startTime) + " ms.)");
     return entities;
   }
+
+  private boolean isDisplayEntityPerUserFilterEnabled(Configuration config) {
+    return config
+        .getBoolean(YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, false);
+  }
+
+  private boolean validateAuthUserWithEntityUser(UserGroupInformation ugi,
+      String entityUser) {
+    String authUser = TimelineReaderWebServicesUtils.getUserName(ugi);
+    String requestedUser = TimelineReaderWebServicesUtils.parseStr(entityUser);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug(
+          "Authenticated User: " + authUser + " Requested User:" + entityUser);
+    }
+    return authUser.equals(requestedUser);
+  }
 }