YARN-8934. [GPG] Add JvmMetricsInfo and pause monitor. Contributed by Bilwa S T.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -3430,6 +3430,14 @@ public class YarnConfiguration extends Configuration {
   public static final String FEDERATION_GPG_PREFIX =
       FEDERATION_PREFIX + "gpg.";
 
+  public static final String GPG_WEBAPP_PREFIX = FEDERATION_GPG_PREFIX
+      + "webapp.";
+
+  /** Enable/disable CORS filter. */
+  public static final String GPG_WEBAPP_ENABLE_CORS_FILTER =
+      GPG_WEBAPP_PREFIX + "cross-origin.enabled";
+  public static final boolean DEFAULT_GPG_WEBAPP_ENABLE_CORS_FILTER = false;
+
   // The number of threads to use for the GPG scheduled executor service
   public static final String GPG_SCHEDULED_EXECUTOR_THREADS =
       FEDERATION_GPG_PREFIX + "scheduled.executor.threads";
@@ -3457,6 +3465,22 @@ public class YarnConfiguration extends Configuration {
       FEDERATION_GPG_PREFIX + "application.cleaner.interval-ms";
   public static final long DEFAULT_GPG_APPCLEANER_INTERVAL_MS = -1;
 
+  /** The address of the GPG web application. */
+  public static final String GPG_WEBAPP_ADDRESS =
+      GPG_WEBAPP_PREFIX + "address";
+
+  public static final int DEFAULT_GPG_WEBAPP_PORT = 8069;
+  public static final String DEFAULT_GPG_WEBAPP_ADDRESS =
+      "0.0.0.0:" + DEFAULT_GPG_WEBAPP_PORT;
+
+  /** The https address of the GPG web application. */
+  public static final String GPG_WEBAPP_HTTPS_ADDRESS =
+      GPG_WEBAPP_PREFIX + "https.address";
+
+  public static final int DEFAULT_GPG_WEBAPP_HTTPS_PORT = 8070;
+  public static final String DEFAULT_GPG_WEBAPP_HTTPS_ADDRESS =
+      "0.0.0.0:" + DEFAULT_GPG_WEBAPP_HTTPS_PORT;
+
   /**
    * Specifications on how (many times) to contact Router for apps. We need to
    * do this because Router might return partial application list because some

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml

@@ -237,6 +237,7 @@
             <exclude>src/main/resources/webapps/test/.keep</exclude>
             <exclude>src/main/resources/webapps/proxy/.keep</exclude>
             <exclude>src/main/resources/webapps/node/.keep</exclude>
+            <exclude>src/main/resources/webapps/gpg/.keep</exclude>
             <exclude>src/main/resources/webapps/static/dt-1.10.18/css/jquery.dataTables.css</exclude>
             <exclude>src/main/resources/webapps/static/dt-1.10.18/css/custom_datatable.css</exclude>
             <exclude>src/main/resources/webapps/static/dt-1.10.18/css/jui-dt.css</exclude>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java

@@ -143,6 +143,16 @@ public class WebAppUtils {
     }
   }
 
+  public static String getGPGWebAppURLWithoutScheme(Configuration conf) {
+    if (YarnConfiguration.useHttps(conf)) {
+      return conf.get(YarnConfiguration.GPG_WEBAPP_HTTPS_ADDRESS,
+          YarnConfiguration.DEFAULT_GPG_WEBAPP_HTTPS_ADDRESS);
+    } else {
+      return conf.get(YarnConfiguration.GPG_WEBAPP_ADDRESS,
+          YarnConfiguration.DEFAULT_GPG_WEBAPP_ADDRESS);
+    }
+  }
+
   public static List<String> getProxyHostsAndPortsForAmFilter(
       Configuration conf) {
     List<String> addrs = new ArrayList<String>();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -3705,6 +3705,34 @@
     <value>3600000</value>
   </property>
 
+  <property>
+    <description>Flag to enable cross-origin (CORS) support in the GPG. This flag
+    requires the CORS filter initializer to be added to the filter initializers
+    list in core-site.xml.</description>
+    <name>yarn.federation.gpg.webapp.cross-origin.enabled</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <description>
+      The http address of the GPG web application.
+      If only a host is provided as the value,
+      the webapp will be served on a random port.
+    </description>
+    <name>yarn.federation.gpg.webapp.address</name>
+    <value>0.0.0.0:8069</value>
+  </property>
+
+  <property>
+    <description>
+      The https address of the GPG web application.
+      If only a host is provided as the value,
+      the webapp will be served on a random port.
+    </description>
+    <name> yarn.federation.gpg.webapp.https.address</name>
+    <value>0.0.0.0:8070</value>
+  </property>
+
   <property>
     <description>
       The configured policy generator class, runs NoOpGlobalPolicy by default

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java

@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.yarn.server.globalpolicygenerator;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -25,9 +27,13 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.commons.lang3.time.DurationFormatUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.source.JvmMetrics;
 import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
+import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.JvmPauseMonitor;
 import org.apache.hadoop.util.ShutdownHookManager;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
@@ -37,9 +43,15 @@ import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade
 import org.apache.hadoop.yarn.server.globalpolicygenerator.applicationcleaner.ApplicationCleaner;
 import org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator.PolicyGenerator;
 import org.apache.hadoop.yarn.server.globalpolicygenerator.subclustercleaner.SubClusterCleaner;
+import org.apache.hadoop.yarn.server.globalpolicygenerator.webapp.GPGWebApp;
+import org.apache.hadoop.yarn.webapp.WebApp;
+import org.apache.hadoop.yarn.webapp.WebApps;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * Global Policy Generator (GPG) is a Yarn Federation component. By tuning the
  * Federation policies in Federation State Store, GPG overlooks the entire
@@ -60,6 +72,7 @@ public class GlobalPolicyGenerator extends CompositeService {
   public static final int SHUTDOWN_HOOK_PRIORITY = 30;
   private AtomicBoolean isStopping = new AtomicBoolean(false);
   private static final String METRICS_NAME = "Global Policy Generator";
+  private static long gpgStartupTime = System.currentTimeMillis();
 
   // Federation Variables
   private GPGContext gpgContext;
@@ -70,6 +83,9 @@ public class GlobalPolicyGenerator extends CompositeService {
   private SubClusterCleaner subClusterCleaner;
   private ApplicationCleaner applicationCleaner;
   private PolicyGenerator policyGenerator;
+  private String webAppAddress;
+  private JvmPauseMonitor pauseMonitor;
+  private WebApp webApp;
 
   public GlobalPolicyGenerator() {
     super(GlobalPolicyGenerator.class.getName());
@@ -110,7 +126,13 @@ public class GlobalPolicyGenerator extends CompositeService {
 
     this.policyGenerator = new PolicyGenerator(conf, this.gpgContext);
 
+    this.webAppAddress = WebAppUtils.getGPGWebAppURLWithoutScheme(conf);
+
     DefaultMetricsSystem.initialize(METRICS_NAME);
+    JvmMetrics jm = JvmMetrics.initSingleton("GPG", null);
+    pauseMonitor = new JvmPauseMonitor();
+    addService(pauseMonitor);
+    jm.setPauseMonitor(pauseMonitor);
 
     // super.serviceInit after all services are added
     super.serviceInit(conf);
@@ -154,6 +176,7 @@ public class GlobalPolicyGenerator extends CompositeService {
       LOG.info("Scheduled policygenerator with interval: {}",
           DurationFormatUtils.formatDurationISO(policyGeneratorIntervalMillis));
     }
+    startWepApp();
   }
 
   @Override
@@ -176,6 +199,9 @@ public class GlobalPolicyGenerator extends CompositeService {
     if (this.isStopping.getAndSet(true)) {
       return;
     }
+    if (webApp != null) {
+      webApp.stop();
+    }
     DefaultMetricsSystem.shutdown();
     super.serviceStop();
   }
@@ -202,6 +228,42 @@ public class GlobalPolicyGenerator extends CompositeService {
     this.start();
   }
 
+  @VisibleForTesting
+  public void startWepApp() {
+    boolean enableCors = getConfig().getBoolean(
+        YarnConfiguration.GPG_WEBAPP_ENABLE_CORS_FILTER,
+        YarnConfiguration.DEFAULT_GPG_WEBAPP_ENABLE_CORS_FILTER);
+    if (enableCors) {
+      getConfig().setBoolean(HttpCrossOriginFilterInitializer.PREFIX
+          + HttpCrossOriginFilterInitializer.ENABLED_SUFFIX, true);
+    }
+
+    // Always load pseudo authentication filter to parse "user.name" in an URL
+    // to identify a HTTP request's user.
+    boolean hasHadoopAuthFilterInitializer = false;
+    String filterInitializerConfKey = "hadoop.http.filter.initializers";
+    Class<?>[] initializersClasses = getConfig()
+        .getClasses(filterInitializerConfKey);
+    List<String> targets = new ArrayList<String>();
+    if (initializersClasses != null) {
+      for (Class<?> initializer : initializersClasses) {
+        if (initializer.getName()
+            .equals(AuthenticationFilterInitializer.class.getName())) {
+          hasHadoopAuthFilterInitializer = true;
+          break;
+        }
+        targets.add(initializer.getName());
+      }
+    }
+    if (!hasHadoopAuthFilterInitializer) {
+      targets.add(AuthenticationFilterInitializer.class.getName());
+      getConfig().set(filterInitializerConfKey, StringUtils.join(",", targets));
+    }
+    LOG.info("Instantiating GPGWebApp at " + webAppAddress);
+    GPGWebApp gpgWebApp = new GPGWebApp(this);
+    webApp = WebApps.$for("gpg").at(webAppAddress).start(gpgWebApp);
+  }
+
   @SuppressWarnings("resource")
   public static void startGPG(String[] argv, Configuration conf) {
     boolean federationEnabled =
@@ -225,6 +287,10 @@ public class GlobalPolicyGenerator extends CompositeService {
     }
   }
 
+  public static long getGPGStartupTime() {
+    return gpgStartupTime;
+  }
+
   public static void main(String[] argv) {
     startGPG(argv, new YarnConfiguration());
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGController.java

@@ -0,0 +1,44 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
+
+import org.apache.hadoop.yarn.webapp.Controller;
+import com.google.inject.Inject;
+
+/**
+ * Controller for the GPG Web UI.
+ */
+public class GPGController extends Controller {
+
+  @Inject
+  GPGController(RequestContext ctx) {
+    super(ctx);
+  }
+
+  @Override
+  public void index() {
+    setTitle("GPG");
+    render(GPGOverviewPage.class);
+  }
+
+  public void overview() {
+    setTitle("GPG Details");
+    render(GPGOverviewPage.class);
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGOverviewBlock.java

@@ -0,0 +1,50 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
+
+import java.util.Date;
+
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator;
+import org.apache.hadoop.yarn.util.YarnVersionInfo;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.hadoop.yarn.webapp.view.InfoBlock;
+
+import com.google.inject.Inject;
+
+/**
+ * Overview block for the GPG Web UI.
+ */
+public class GPGOverviewBlock extends HtmlBlock {
+
+  @Inject
+  GPGOverviewBlock(GlobalPolicyGenerator gpg, ViewContext ctx) {
+    super(ctx);
+  }
+
+  @Override
+  protected void render(Block html) {
+    info("GPG Details")
+        .__("GPG started on",
+            new Date(GlobalPolicyGenerator.getGPGStartupTime()))
+        .__("GPG Version", YarnVersionInfo.getVersion())
+        .__("Hadoop Version", VersionInfo.getVersion());
+
+    html.__(InfoBlock.class);
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGOverviewPage.java

@@ -0,0 +1,52 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
+
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout;
+
+/**
+ * Overview page for the GPG Web UI.
+ */
+public class GPGOverviewPage extends TwoColumnLayout {
+
+  @Override
+  protected void preHead(Page.HTML<__> html) {
+    commonPreHead(html);
+    setTitle("GPG");
+  }
+
+  protected void commonPreHead(Page.HTML<__> html) {
+    set(ACCORDION_ID, "nav");
+    set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}");
+  }
+
+  @Override
+  protected Class<? extends SubView> nav() {
+    return NavBlock.class;
+  }
+
+  @Override
+  protected Class<? extends SubView> content() {
+    return GPGOverviewBlock.class;
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java

@@ -0,0 +1,45 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
+
+import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.WebApp;
+
+/**
+ * The GPG webapp.
+ */
+public class GPGWebApp extends WebApp{
+  private GlobalPolicyGenerator gpg;
+
+  public GPGWebApp(GlobalPolicyGenerator gpg) {
+    this.gpg = gpg;
+  }
+
+  @Override
+  public void setup() {
+    bind(JAXBContextResolver.class);
+    bind(GPGWebApp.class).toInstance(this);
+    bind(GenericExceptionHandler.class);
+    if (gpg != null) {
+      bind(GlobalPolicyGenerator.class).toInstance(gpg);
+    }
+    route("/", GPGController.class, "overview");
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/NavBlock.java

@@ -0,0 +1,42 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
+
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+
+/**
+ * Navigation block for the GPG Web UI.
+ */
+public class NavBlock extends HtmlBlock {
+
+  @Override
+  public void render(Block html) {
+    html.
+      div("#nav").
+        h3("GPG").
+        ul().
+          li().a(url(""), "Overview").__().
+        __().
+        h3("Tools").
+        ul().
+          li().a("/conf", "Configuration").__().
+          li().a("/logs", "Local logs").__().
+          li().a("/stacks", "Server stacks").__().
+          li().a("/jmx?qry=Hadoop:*", "Server metrics").__().__().__();
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/package-info.java

@@ -0,0 +1,24 @@
+/**
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Classes comprising the policy generator for the GPG. Responsibilities include
+ * generating and updating policies based on the cluster status.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;