YARN-11345. [Federation] Refactoring Yarn Router's Application Web Page. (#5030)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java

@@ -35,6 +35,7 @@ public interface YarnWebParams {
   String APP_STATE = "app.state";
   String APP_START_TIME_BEGIN = "app.started-time.begin";
   String APP_START_TIME_END = "app.started-time.end";
+  String APP_SC = "app.subcluster";
   String APPS_NUM = "apps.num";
   String QUEUE_NAME = "queue.name";
   String NODE_STATE = "node.state";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java

@@ -21,11 +21,18 @@ package org.apache.hadoop.yarn.server.router.webapp;
 import static org.apache.commons.text.StringEscapeUtils.escapeHtml4;
 import static org.apache.commons.text.StringEscapeUtils.escapeEcmaScript;
 import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_SC;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
 
 import com.sun.jersey.api.client.Client;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
@@ -34,34 +41,102 @@ import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
 import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
 import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
 
 import com.google.inject.Inject;
 
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
 /**
  * Applications block for the Router Web UI.
  */
-public class AppsBlock extends HtmlBlock {
+public class AppsBlock extends RouterBlock {
+
   private final Router router;
+  private final Configuration conf;
 
   @Inject
   AppsBlock(Router router, ViewContext ctx) {
-    super(ctx);
+    super(router, ctx);
     this.router = router;
+    this.conf = this.router.getConfig();
   }
 
   @Override
   protected void render(Block html) {
-    // Get the applications from the Resource Managers
-    Configuration conf = this.router.getConfig();
+
+    boolean isEnabled = isYarnFederationEnabled();
+
+    // Get subClusterName
+    String subClusterName = $(APP_SC);
+    String reqState = $(APP_STATE);
+
+    // We will try to get the subClusterName.
+    // If the subClusterName is not empty,
+    // it means that we need to get the Node list of a subCluster.
+    AppsInfo appsInfo = null;
+    if (subClusterName != null && !subClusterName.isEmpty()) {
+      initSubClusterMetricsOverviewTable(html, subClusterName);
+      appsInfo = getSubClusterAppsInfo(subClusterName, reqState);
+    } else {
+      // Metrics Overview Table
+      html.__(MetricsOverviewTable.class);
+      appsInfo = getYarnFederationAppsInfo(isEnabled);
+    }
+
+    initYarnFederationAppsOfCluster(appsInfo, html);
+  }
+
+  private static String escape(String str) {
+    return escapeEcmaScript(escapeHtml4(str));
+  }
+
+  private AppsInfo getYarnFederationAppsInfo(boolean isEnabled) {
+    if (isEnabled) {
+      String webAddress = WebAppUtils.getRouterWebAppURLWithScheme(this.conf);
+      return getSubClusterAppsInfoByWebAddress(webAddress, StringUtils.EMPTY);
+    }
+    return null;
+  }
+
+  private AppsInfo getSubClusterAppsInfo(String subCluster, String states) {
+    try {
+      SubClusterId subClusterId = SubClusterId.newInstance(subCluster);
+      FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
+      SubClusterInfo subClusterInfo = facade.getSubCluster(subClusterId);
+
+      if (subClusterInfo != null) {
+        // Prepare webAddress
+        String webAddress = subClusterInfo.getRMWebServiceAddress();
+        String herfWebAppAddress = "";
+        if (webAddress != null && !webAddress.isEmpty()) {
+          herfWebAppAddress = WebAppUtils.getHttpSchemePrefix(conf) + webAddress;
+          return getSubClusterAppsInfoByWebAddress(herfWebAppAddress, states);
+        }
+      }
+    } catch (Exception e) {
+      LOG.error("get AppsInfo From SubCluster = {} error.", subCluster, e);
+    }
+    return null;
+  }
+
+  private AppsInfo getSubClusterAppsInfoByWebAddress(String webAddress, String states) {
     Client client = RouterWebServiceUtil.createJerseyClient(conf);
-    String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
+    Map<String, String[]> queryParams = new HashMap<>();
+    if (StringUtils.isNotBlank(states)) {
+      queryParams.put("states", new String[]{states});
+    }
     AppsInfo apps = RouterWebServiceUtil
-        .genericForward(webAppAddress, null, AppsInfo.class, HTTPMethods.GET,
-            RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null, conf,
-            client);
+        .genericForward(webAddress, null, AppsInfo.class, HTTPMethods.GET,
+        RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, queryParams, conf,
+        client);
+    client.destroy();
+    return apps;
+  }
 
-    setTitle("Applications");
+  private void initYarnFederationAppsOfCluster(AppsInfo appsInfo, Block html) {
 
     TBODY<TABLE<Hamlet>> tbody = html.table("#apps").thead()
         .tr()
@@ -81,45 +156,18 @@ public class AppsBlock extends HtmlBlock {
 
     // Render the applications
     StringBuilder appsTableData = new StringBuilder("[\n");
-    for (AppInfo app : apps.getApps()) {
-      try {
-
-        String percent = String.format("%.1f", app.getProgress() * 100.0F);
-        String trackingURL =
-            app.getTrackingUrl() == null ? "#" : app.getTrackingUrl();
-        // AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
-        appsTableData.append("[\"")
-            .append("<a href='").append(trackingURL).append("'>")
-            .append(app.getAppId()).append("</a>\",\"")
-            .append(escape(app.getUser())).append("\",\"")
-            .append(escape(app.getName())).append("\",\"")
-            .append(escape(app.getApplicationType())).append("\",\"")
-            .append(escape(app.getQueue())).append("\",\"")
-            .append(String.valueOf(app.getPriority())).append("\",\"")
-            .append(app.getStartTime()).append("\",\"")
-            .append(app.getFinishTime()).append("\",\"")
-            .append(app.getState()).append("\",\"")
-            .append(app.getFinalStatus()).append("\",\"")
-            // Progress bar
-            .append("<br title='").append(percent).append("'> <div class='")
-            .append(C_PROGRESSBAR).append("' title='")
-            .append(join(percent, '%')).append("'> ").append("<div class='")
-            .append(C_PROGRESSBAR_VALUE).append("' style='")
-            .append(join("width:", percent, '%')).append("'> </div> </div>")
-            // History link
-            .append("\",\"<a href='").append(trackingURL).append("'>")
-            .append("History").append("</a>");
-        appsTableData.append("\"],\n");
-
-      } catch (Exception e) {
-        LOG.info(
-            "Cannot add application {}: {}", app.getAppId(), e.getMessage());
+
+    if (appsInfo != null && CollectionUtils.isNotEmpty(appsInfo.getApps())) {
+
+      List<String> appInfoList =
+          appsInfo.getApps().stream().map(this::parseAppInfoData).collect(Collectors.toList());
+
+      if (CollectionUtils.isNotEmpty(appInfoList)) {
+        String formattedAppInfo = StringUtils.join(appInfoList, ",");
+        appsTableData.append(formattedAppInfo);
       }
     }
-    if (appsTableData.charAt(appsTableData.length() - 2) == ',') {
-      appsTableData.delete(appsTableData.length() - 2,
-          appsTableData.length() - 1);
-    }
+
     appsTableData.append("]");
     html.script().$type("text/javascript")
         .__("var appsTableData=" + appsTableData).__();
@@ -127,7 +175,39 @@ public class AppsBlock extends HtmlBlock {
     tbody.__().__();
   }
 
-  private static String escape(String str) {
-    return escapeEcmaScript(escapeHtml4(str));
+  private String parseAppInfoData(AppInfo app) {
+    StringBuilder appsDataBuilder = new StringBuilder();
+    try {
+      String percent = String.format("%.1f", app.getProgress() * 100.0F);
+      String trackingURL = app.getTrackingUrl() == null ? "#" : app.getTrackingUrl();
+
+      // AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
+      appsDataBuilder.append("[\"")
+          .append("<a href='").append(trackingURL).append("'>")
+          .append(app.getAppId()).append("</a>\",\"")
+          .append(escape(app.getUser())).append("\",\"")
+          .append(escape(app.getName())).append("\",\"")
+          .append(escape(app.getApplicationType())).append("\",\"")
+          .append(escape(app.getQueue())).append("\",\"")
+          .append(app.getPriority()).append("\",\"")
+          .append(app.getStartTime()).append("\",\"")
+          .append(app.getFinishTime()).append("\",\"")
+          .append(app.getState()).append("\",\"")
+          .append(app.getFinalStatus()).append("\",\"")
+          // Progress bar
+          .append("<br title='").append(percent).append("'> <div class='")
+          .append(C_PROGRESSBAR).append("' title='")
+          .append(join(percent, '%')).append("'> ").append("<div class='")
+          .append(C_PROGRESSBAR_VALUE).append("' style='")
+          .append(join("width:", percent, '%')).append("'> </div> </div>")
+          // History link
+          .append("\",\"<a href='").append(trackingURL).append("'>")
+          .append("History").append("</a>");
+      appsDataBuilder.append("\"]\n");
+
+    } catch (Exception e) {
+      LOG.warn("Cannot add application {}: {}", app.getAppId(), e.getMessage());
+    }
+    return appsDataBuilder.toString();
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java

@@ -20,11 +20,13 @@ package org.apache.hadoop.yarn.server.router.webapp;
 
 import static org.apache.hadoop.yarn.util.StringHelper.sjoin;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_SC;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.yarn.webapp.SubView;
 
 class AppsPage extends RouterView {
@@ -37,9 +39,14 @@ class AppsPage extends RouterView {
     setTableStyles(html, "apps", ".queue {width:6em}", ".ui {width:8em}");
 
     // Set the correct title.
+    String subClusterName = $(APP_SC);
     String reqState = $(APP_STATE);
-    reqState = (reqState == null || reqState.isEmpty() ? "All" : reqState);
-    setTitle(sjoin(reqState, "Applications"));
+
+    if(StringUtils.isBlank(subClusterName)){
+      subClusterName = "Federation ";
+    }
+    reqState = (StringUtils.isBlank(reqState) ? "All" : reqState);
+    setTitle(sjoin(subClusterName, reqState,  "Applications"));
   }
 
   private String appsTableInit() {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java

@@ -20,12 +20,12 @@ package org.apache.hadoop.yarn.server.router.webapp;
 
 import com.google.inject.Inject;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.server.router.Router;
 import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
 import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
 
 import java.util.List;
-
 /**
  * Navigation block for the Router Web UI.
  */
@@ -60,7 +60,24 @@ public class NavBlock extends RouterBlock {
     subAppsList1.__().__();
 
     // ### applications info
-    mainList.li().a(url("apps"), "Applications").__();
+    Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.DIV<Hamlet>>>> subAppsList2 =
+            mainList.li().a(url("apps"), "Applications").ul();
+
+    subAppsList2.li().__();
+    for (String subClusterId : subClusterIds) {
+      Hamlet.LI<Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.DIV<Hamlet>>>>> subAppsList3 = subAppsList2.
+          li().a(url("apps", subClusterId), subClusterId);
+      Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.DIV<Hamlet>>>>>> subAppsList4 =
+          subAppsList3.ul().$style("padding:0.3em 1em 0.1em 2em");
+      subAppsList4.li().__();
+      for (YarnApplicationState state : YarnApplicationState.values()) {
+        subAppsList4.
+            li().a(url("apps", subClusterId, state.toString()), state.toString()).__();
+      }
+      subAppsList4.li().__().__();
+      subAppsList3.__();
+    }
+    subAppsList2.__().__();
 
     // ### tools
     Hamlet.DIV<Hamlet> sectionBefore = mainList.__();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java

@@ -116,6 +116,7 @@ public class NodesBlock extends RouterBlock {
         .genericForward(webAddress, null, NodesInfo.class, HTTPMethods.GET,
         RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null, conf,
         client);
+    client.destroy();
     return nodes;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java

@@ -49,7 +49,7 @@ public class RouterWebApp extends WebApp implements YarnWebParams {
     route("/", RouterController.class);
     route("/cluster", RouterController.class, "about");
     route("/about", RouterController.class, "about");
-    route("/apps", RouterController.class, "apps");
+    route(pajoin("/apps", APP_SC, APP_STATE), RouterController.class, "apps");
     route(pajoin("/nodes", NODE_SC), RouterController.class, "nodes");
     route("/federation", RouterController.class, "federation");
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java

@@ -81,4 +81,22 @@ public class TestFederationWebApp extends TestRouterWebServicesREST {
     config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
     WebAppTests.testPage(NodesPage.class, Router.class, new MockRouter(config));
   }
+
+  @Test
+  public void testFederationAppViewEnable()
+      throws InterruptedException, YarnException, IOException {
+    // Test Federation Enabled
+    Configuration config = new YarnConfiguration();
+    config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+    WebAppTests.testPage(AppsPage.class, Router.class, new MockRouter(config));
+  }
+
+  @Test
+  public void testFederationAppViewNotEnable()
+      throws InterruptedException, YarnException, IOException {
+    // Test Federation Not Enabled
+    Configuration config = new YarnConfiguration();
+    config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
+    WebAppTests.testPage(AppsPage.class, Router.class, new MockRouter(config));
+  }
 }