YARN-8934. [BackPort][GPG] Add JvmMetricsInfo and pause monitor. (#5929) Contributed by Bilwa S T.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -4445,6 +4445,27 @@ public class YarnConfiguration extends Configuration {
       FEDERATION_GPG_LOAD_BASED_PREFIX + "scaling";
   public static final String DEFAULT_FEDERATION_GPG_LOAD_BASED_SCALING = "LINEAR";
 
+  public static final String GPG_WEBAPP_PREFIX = FEDERATION_GPG_PREFIX + "webapp.";
+
+  /** Enable/disable CORS filter. */
+  public static final String GPG_WEBAPP_ENABLE_CORS_FILTER =
+      GPG_WEBAPP_PREFIX + "cross-origin.enabled";
+  public static final boolean DEFAULT_GPG_WEBAPP_ENABLE_CORS_FILTER = false;
+
+  /** The address of the GPG web application. */
+  public static final String GPG_WEBAPP_ADDRESS = GPG_WEBAPP_PREFIX + "address";
+
+  public static final int DEFAULT_GPG_WEBAPP_PORT = 8069;
+  public static final String DEFAULT_GPG_WEBAPP_ADDRESS =
+      "0.0.0.0:" + DEFAULT_GPG_WEBAPP_PORT;
+
+  /** The https address of the GPG web application. */
+  public static final String GPG_WEBAPP_HTTPS_ADDRESS = GPG_WEBAPP_PREFIX + "https.address";
+
+  public static final int DEFAULT_GPG_WEBAPP_HTTPS_PORT = 8070;
+  public static final String DEFAULT_GPG_WEBAPP_HTTPS_ADDRESS =
+      "0.0.0.0:" + DEFAULT_GPG_WEBAPP_HTTPS_PORT;
+
   /**
    * Connection and Read timeout from the Router to RM.
    */

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml

@@ -271,6 +271,7 @@
             <exclude>src/main/resources/webapps/test/.keep</exclude>
             <exclude>src/main/resources/webapps/proxy/.keep</exclude>
             <exclude>src/main/resources/webapps/node/.keep</exclude>
+            <exclude>src/main/resources/webapps/gpg/.keep</exclude>
             <exclude>src/main/resources/webapps/static/dt-1.11.5/css/jquery.dataTables.css</exclude>
             <exclude>src/main/resources/webapps/static/dt-1.11.5/css/custom_datatable.css</exclude>
             <exclude>src/main/resources/webapps/static/dt-1.11.5/css/jui-dt.css</exclude>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java

@@ -183,6 +183,16 @@ public class WebAppUtils {
     }
   }
 
+  public static String getGPGWebAppURLWithoutScheme(Configuration conf) {
+    if (YarnConfiguration.useHttps(conf)) {
+      return conf.get(YarnConfiguration.GPG_WEBAPP_HTTPS_ADDRESS,
+          YarnConfiguration.DEFAULT_GPG_WEBAPP_HTTPS_ADDRESS);
+    } else {
+      return conf.get(YarnConfiguration.GPG_WEBAPP_ADDRESS,
+          YarnConfiguration.DEFAULT_GPG_WEBAPP_ADDRESS);
+    }
+  }
+
   public static List<String> getProxyHostsAndPortsForAmFilter(
       Configuration conf) {
     List<String> addrs = new ArrayList<String>();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -5530,4 +5530,32 @@
     <value>LINEAR</value>
   </property>
 
+  <property>
+    <description>Flag to enable cross-origin (CORS) support in the GPG. This flag
+      requires the CORS filter initializer to be added to the filter initializers
+      list in core-site.xml.</description>
+    <name>yarn.federation.gpg.webapp.cross-origin.enabled</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <description>
+      The http address of the GPG web application.
+      If only a host is provided as the value,
+      the webapp will be served on a random port.
+    </description>
+    <name>yarn.federation.gpg.webapp.address</name>
+    <value>0.0.0.0:8069</value>
+  </property>
+
+  <property>
+    <description>
+      The https address of the GPG web application.
+      If only a host is provided as the value,
+      the webapp will be served on a random port.
+    </description>
+    <name> yarn.federation.gpg.webapp.https.address</name>
+    <value>0.0.0.0:8070</value>
+  </property>
+
 </configuration>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java

@@ -21,15 +21,22 @@ package org.apache.hadoop.yarn.server.globalpolicygenerator;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.lang.time.DurationFormatUtils;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
+import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.JvmPauseMonitor;
 import org.apache.hadoop.util.ShutdownHookManager;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
@@ -38,6 +45,10 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
 import org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator.PolicyGenerator;
 import org.apache.hadoop.yarn.server.globalpolicygenerator.subclustercleaner.SubClusterCleaner;
+import org.apache.hadoop.yarn.server.globalpolicygenerator.webapp.GPGWebApp;
+import org.apache.hadoop.yarn.webapp.WebApp;
+import org.apache.hadoop.yarn.webapp.WebApps;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -61,6 +72,7 @@ public class GlobalPolicyGenerator extends CompositeService {
   public static final int SHUTDOWN_HOOK_PRIORITY = 30;
   private AtomicBoolean isStopping = new AtomicBoolean(false);
   private static final String METRICS_NAME = "Global Policy Generator";
+  private static long gpgStartupTime = System.currentTimeMillis();
 
   // Federation Variables
   private GPGContext gpgContext;
@@ -69,6 +81,9 @@ public class GlobalPolicyGenerator extends CompositeService {
   private ScheduledThreadPoolExecutor scheduledExecutorService;
   private SubClusterCleaner subClusterCleaner;
   private PolicyGenerator policyGenerator;
+  private String webAppAddress;
+  private JvmPauseMonitor pauseMonitor;
+  private WebApp webApp;
 
   public GlobalPolicyGenerator() {
     super(GlobalPolicyGenerator.class.getName());
@@ -107,7 +122,12 @@ public class GlobalPolicyGenerator extends CompositeService {
     this.subClusterCleaner = new SubClusterCleaner(conf, this.gpgContext);
     this.policyGenerator = new PolicyGenerator(conf, this.gpgContext);
 
+    this.webAppAddress = WebAppUtils.getGPGWebAppURLWithoutScheme(conf);
     DefaultMetricsSystem.initialize(METRICS_NAME);
+    JvmMetrics jm = JvmMetrics.initSingleton("GPG", null);
+    pauseMonitor = new JvmPauseMonitor();
+    addService(pauseMonitor);
+    jm.setPauseMonitor(pauseMonitor);
 
     // super.serviceInit after all services are added
     super.serviceInit(conf);
@@ -163,6 +183,7 @@ public class GlobalPolicyGenerator extends CompositeService {
       LOG.info("Scheduled policy-generator with interval: {}",
           DurationFormatUtils.formatDurationISO(policyGeneratorIntervalMillis));
     }
+    startWepApp();
   }
 
   @Override
@@ -181,6 +202,9 @@ public class GlobalPolicyGenerator extends CompositeService {
     if (this.isStopping.getAndSet(true)) {
       return;
     }
+    if (webApp != null) {
+      webApp.stop();
+    }
     DefaultMetricsSystem.shutdown();
     super.serviceStop();
   }
@@ -193,6 +217,43 @@ public class GlobalPolicyGenerator extends CompositeService {
     return this.gpgContext;
   }
 
+  @VisibleForTesting
+  public void startWepApp() {
+    Configuration configuration = getConfig();
+
+    boolean enableCors = configuration.getBoolean(YarnConfiguration.GPG_WEBAPP_ENABLE_CORS_FILTER,
+        YarnConfiguration.DEFAULT_GPG_WEBAPP_ENABLE_CORS_FILTER);
+
+    if (enableCors) {
+      configuration.setBoolean(HttpCrossOriginFilterInitializer.PREFIX
+          + HttpCrossOriginFilterInitializer.ENABLED_SUFFIX, true);
+    }
+
+    // Always load pseudo authentication filter to parse "user.name" in an URL
+    // to identify a HTTP request's user.
+    boolean hasHadoopAuthFilterInitializer = false;
+    String filterInitializerConfKey = "hadoop.http.filter.initializers";
+    Class<?>[] initializersClasses = configuration.getClasses(filterInitializerConfKey);
+
+    List<String> targets = new ArrayList<>();
+    if (initializersClasses != null) {
+      for (Class<?> initializer : initializersClasses) {
+        if (initializer.getName().equals(AuthenticationFilterInitializer.class.getName())) {
+          hasHadoopAuthFilterInitializer = true;
+          break;
+        }
+        targets.add(initializer.getName());
+      }
+    }
+    if (!hasHadoopAuthFilterInitializer) {
+      targets.add(AuthenticationFilterInitializer.class.getName());
+      configuration.set(filterInitializerConfKey, StringUtils.join(",", targets));
+    }
+    LOG.info("Instantiating GPGWebApp at {}.", webAppAddress);
+    GPGWebApp gpgWebApp = new GPGWebApp(this);
+    webApp = WebApps.$for("gpg").at(webAppAddress).start(gpgWebApp);
+  }
+
   @SuppressWarnings("resource")
   public static void startGPG(String[] argv, Configuration conf) {
     boolean federationEnabled = conf.getBoolean(YarnConfiguration.FEDERATION_ENABLED,
@@ -232,4 +293,13 @@ public class GlobalPolicyGenerator extends CompositeService {
       System.exit(-1);
     }
   }
+
+  public static long getGPGStartupTime() {
+    return gpgStartupTime;
+  }
+
+  @VisibleForTesting
+  public WebApp getWebApp() {
+    return webApp;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGController.java

@@ -0,0 +1,44 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
+
+import org.apache.hadoop.yarn.webapp.Controller;
+import com.google.inject.Inject;
+
+/**
+ * Controller for the GPG Web UI.
+ */
+public class GPGController extends Controller {
+
+  @Inject
+  GPGController(RequestContext ctx) {
+    super(ctx);
+  }
+
+  @Override
+  public void index() {
+    setTitle("GPG");
+    render(GPGOverviewPage.class);
+  }
+
+  public void overview() {
+    setTitle("GPG Details");
+    render(GPGOverviewPage.class);
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGOverviewBlock.java

@@ -0,0 +1,49 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
+
+import java.util.Date;
+
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator;
+import org.apache.hadoop.yarn.util.YarnVersionInfo;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.hadoop.yarn.webapp.view.InfoBlock;
+
+import com.google.inject.Inject;
+
+/**
+ * Overview block for the GPG Web UI.
+ */
+public class GPGOverviewBlock extends HtmlBlock {
+
+  @Inject
+  GPGOverviewBlock(GlobalPolicyGenerator gpg, ViewContext ctx) {
+    super(ctx);
+  }
+
+  @Override
+  protected void render(Block html) {
+    info("GPG Details")
+        .__("GPG started on", new Date(GlobalPolicyGenerator.getGPGStartupTime()))
+        .__("GPG Version", YarnVersionInfo.getVersion())
+        .__("Hadoop Version", VersionInfo.getVersion());
+
+    html.__(InfoBlock.class);
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGOverviewPage.java

@@ -0,0 +1,52 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
+
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout;
+
+/**
+ * Overview page for the GPG Web UI.
+ */
+public class GPGOverviewPage extends TwoColumnLayout {
+
+  @Override
+  protected void preHead(Page.HTML<__> html) {
+    commonPreHead(html);
+    setTitle("GPG");
+  }
+
+  protected void commonPreHead(Page.HTML<__> html) {
+    set(ACCORDION_ID, "nav");
+    set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}");
+  }
+
+  @Override
+  protected Class<? extends SubView> nav() {
+    return NavBlock.class;
+  }
+
+  @Override
+  protected Class<? extends SubView> content() {
+    return GPGOverviewBlock.class;
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java

@@ -0,0 +1,45 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
+
+import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.WebApp;
+
+/**
+ * The GPG webapp.
+ */
+public class GPGWebApp extends WebApp{
+  private GlobalPolicyGenerator gpg;
+
+  public GPGWebApp(GlobalPolicyGenerator gpg) {
+    this.gpg = gpg;
+  }
+
+  @Override
+  public void setup() {
+    bind(JAXBContextResolver.class);
+    bind(GPGWebApp.class).toInstance(this);
+    bind(GenericExceptionHandler.class);
+    if (gpg != null) {
+      bind(GlobalPolicyGenerator.class).toInstance(gpg);
+    }
+    route("/", GPGController.class, "overview");
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/NavBlock.java

@@ -0,0 +1,42 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
+
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+
+/**
+ * Navigation block for the GPG Web UI.
+ */
+public class NavBlock extends HtmlBlock {
+
+  @Override
+  public void render(Block html) {
+    html.
+      div("#nav").
+        h3("GPG").
+        ul().
+          li().a(url(""), "Overview").__().
+        __().
+        h3("Tools").
+        ul().
+          li().a("/conf", "Configuration").__().
+          li().a("/logs", "Local logs").__().
+          li().a("/stacks", "Server stacks").__().
+          li().a("/jmx?qry=Hadoop:*", "Server metrics").__().__().__();
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/package-info.java

@@ -0,0 +1,24 @@
+/**
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Classes comprising the policy generator for the GPG. Responsibilities include
+ * generating and updating policies based on the cluster status.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java

@@ -19,9 +19,14 @@
 package org.apache.hadoop.yarn.server.globalpolicygenerator;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.junit.Test;
 
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
 /**
  * Unit test for GlobalPolicyGenerator.
  */
@@ -35,4 +40,19 @@ public class TestGlobalPolicyGenerator {
     // If GPG starts running, this call will not return
     GlobalPolicyGenerator.startGPG(new String[0], conf);
   }
+
+  @Test
+  public void testGpgWithFederation() throws InterruptedException, TimeoutException {
+    // In this test case, we hope that gpg can start normally in federation mode.
+    Configuration conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+
+    GlobalPolicyGenerator gpg = new GlobalPolicyGenerator();
+    gpg.initAndStart(conf, false);
+
+    GenericTestUtils.waitFor(() -> {
+      List<Service> services = gpg.getServices();
+      return (services.size() == 1 && gpg.getWebApp() != null);
+    }, 100, 5000);
+  }
 }