AMBARI-8808. Resolve potential port conflicts with Hadoop daemons. (swagle)

ambari-metrics/ambari-metrics-flume-sink/src/test/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java

@@ -27,10 +27,10 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
-
 import java.util.Collections;
-
-import static org.powermock.api.easymock.PowerMock.*;
+import static org.powermock.api.easymock.PowerMock.mockStatic;
+import static org.powermock.api.easymock.PowerMock.replay;
+import static org.powermock.api.easymock.PowerMock.verifyAll;
 
 @RunWith(PowerMockRunner.class)
 @PrepareForTest(JMXPollUtil.class)

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java

@@ -63,6 +63,7 @@ public class ApplicationHistoryServer extends CompositeService {
   TimelineStore timelineStore;
   TimelineMetricStore timelineMetricStore;
   private WebApp webApp;
+  private TimelineMetricConfiguration metricConfiguration;
 
   public ApplicationHistoryServer() {
     super(ApplicationHistoryServer.class.getName());
@@ -70,6 +71,8 @@ public class ApplicationHistoryServer extends CompositeService {
 
   @Override
   protected void serviceInit(Configuration conf) throws Exception {
+    metricConfiguration = new TimelineMetricConfiguration();
+    metricConfiguration.initialize();
     historyManager = createApplicationHistory();
     ahsClientService = createApplicationHistoryClientService(historyManager);
     addService(ahsClientService);
@@ -162,11 +165,11 @@ public class ApplicationHistoryServer extends CompositeService {
 
   protected TimelineMetricStore createTimelineMetricStore(Configuration conf) {
     LOG.info("Creating metrics store.");
-    return ReflectionUtils.newInstance(HBaseTimelineMetricStore.class, conf);
+    return new HBaseTimelineMetricStore(metricConfiguration);
   }
 
   protected void startWebApp() {
-    String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(getConfig());
+    String bindAddress = metricConfiguration.getWebappAddress();
     LOG.info("Instantiating AHSWebApp at " + bindAddress);
     try {
       webApp =
@@ -186,6 +189,7 @@ public class ApplicationHistoryServer extends CompositeService {
       throw new YarnRuntimeException(msg, e);
     }
   }
+
   /**
    * @return ApplicationTimelineStore
    */

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java

@@ -47,43 +47,22 @@ public class HBaseTimelineMetricStore extends AbstractService
     implements TimelineMetricStore {
 
   static final Log LOG = LogFactory.getLog(HBaseTimelineMetricStore.class);
+  private final TimelineMetricConfiguration configuration;
   private PhoenixHBaseAccessor hBaseAccessor;
 
   /**
    * Construct the service.
    *
    */
-  public HBaseTimelineMetricStore() {
+  public HBaseTimelineMetricStore(TimelineMetricConfiguration configuration) {
     super(HBaseTimelineMetricStore.class.getName());
+    this.configuration = configuration;
   }
 
   @Override
   protected void serviceInit(Configuration conf) throws Exception {
-    ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
-    if (classLoader == null) {
-      classLoader = getClass().getClassLoader();
-    }
-    URL hbaseResUrl = classLoader.getResource(HBASE_SITE_CONFIGURATION_FILE);
-    URL amsResUrl = classLoader.getResource(METRICS_SITE_CONFIGURATION_FILE);
-    LOG.info("Found hbase site configuration: " + hbaseResUrl);
-    LOG.info("Found metric service configuration: " + amsResUrl);
-
-    if (hbaseResUrl == null) {
-      throw new IllegalStateException("Unable to initialize the metrics " +
-        "subsystem. No hbase-site present in the classpath.");
-    }
-
-    if (amsResUrl == null) {
-      throw new IllegalStateException("Unable to initialize the metrics " +
-        "subsystem. No ams-site present in the classpath.");
-    }
-
-    Configuration hbaseConf = new Configuration(true);
-    hbaseConf.addResource(hbaseResUrl.toURI().toURL());
-    Configuration metricsConf = new Configuration(true);
-    metricsConf.addResource(amsResUrl.toURI().toURL());
-
-    initializeSubsystem(hbaseConf, metricsConf);
+    super.serviceInit(conf);
+    initializeSubsystem(configuration.getHbaseConf(), configuration.getMetricsConf());
   }
 
   private void initializeSubsystem(Configuration hbaseConf,
@@ -131,7 +110,6 @@ public class HBaseTimelineMetricStore extends AbstractService
     super.serviceStop();
   }
 
-  //TODO: update to work with HOSTS_COUNT and METRIC_COUNT
   @Override
   public TimelineMetrics getTimelineMetrics(List<String> metricNames,
       String hostname, String applicationId, String instanceId,

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java

@@ -17,8 +17,15 @@
  */
 package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
 
 /**
  * Configuration class that reads properties from ams-site.xml. All values
@@ -26,7 +33,9 @@ import org.apache.hadoop.classification.InterfaceStability;
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
-public interface TimelineMetricConfiguration {
+public class TimelineMetricConfiguration {
+  private static final Log LOG = LogFactory.getLog(TimelineMetricConfiguration.class);
+
   public static final String HBASE_SITE_CONFIGURATION_FILE = "hbase-site.xml";
   public static final String METRICS_SITE_CONFIGURATION_FILE = "ams-site.xml";
 
@@ -112,4 +121,60 @@ public interface TimelineMetricConfiguration {
 
   public static final String DISABLE_APPLICATION_TIMELINE_STORE =
     "timeline.service.disable.application.timeline.store";
+
+  public static final String WEBAPP_HTTP_ADDRESS =
+    "timeline.metrics.service.webapp.address";
+
+  private Configuration hbaseConf;
+  private Configuration metricsConf;
+  private volatile boolean isInitialized = false;
+
+  public void initialize() throws URISyntaxException, MalformedURLException {
+    ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+    if (classLoader == null) {
+      classLoader = getClass().getClassLoader();
+    }
+    URL hbaseResUrl = classLoader.getResource(HBASE_SITE_CONFIGURATION_FILE);
+    URL amsResUrl = classLoader.getResource(METRICS_SITE_CONFIGURATION_FILE);
+    LOG.info("Found hbase site configuration: " + hbaseResUrl);
+    LOG.info("Found metric service configuration: " + amsResUrl);
+
+    if (hbaseResUrl == null) {
+      throw new IllegalStateException("Unable to initialize the metrics " +
+        "subsystem. No hbase-site present in the classpath.");
+    }
+
+    if (amsResUrl == null) {
+      throw new IllegalStateException("Unable to initialize the metrics " +
+        "subsystem. No ams-site present in the classpath.");
+    }
+
+    hbaseConf = new Configuration(true);
+    hbaseConf.addResource(hbaseResUrl.toURI().toURL());
+    metricsConf = new Configuration(true);
+    metricsConf.addResource(amsResUrl.toURI().toURL());
+    isInitialized = true;
+  }
+
+  public Configuration getHbaseConf() throws URISyntaxException, MalformedURLException {
+    if (!isInitialized) {
+      initialize();
+    }
+    return hbaseConf;
+  }
+
+  public Configuration getMetricsConf() throws URISyntaxException, MalformedURLException {
+    if (!isInitialized) {
+      initialize();
+    }
+    return metricsConf;
+  }
+
+  public String getWebappAddress() {
+    String defaultHttpAddress = "0.0.0.0:8188";
+    if (metricsConf != null) {
+      return metricsConf.get(WEBAPP_HTTP_ADDRESS, defaultHttpAddress);
+    }
+    return defaultHttpAddress;
+  }
 }

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java

@@ -98,6 +98,7 @@ public abstract class AbstractProviderModule implements ProviderModule,
   private static final Map<String, Map<String, String[]>> jmxDesiredProperties = new HashMap<String, Map<String, String[]>>();
   private volatile Map<String, String> clusterCoreSiteConfigVersionMap = new HashMap<String, String>();
   private volatile Map<String, String> clusterJmxProtocolMap = new HashMap<String, String>();
+  private volatile String clusterMetricServerPort = "80";
 
   static {
     serviceConfigTypes.put(Service.Type.HDFS, "hdfs-site");
@@ -334,6 +335,7 @@ public abstract class AbstractProviderModule implements ProviderModule,
     if (service.equals(GANGLIA)) {
       return "80"; // Not called by the provider
     } else if (service.equals(TIMELINE_METRICS)) {
+      String collectorPort = null;
       try {
         String configType = serviceConfigTypes.get(Service.Type.AMS);
         String currentConfigVersion = getDesiredConfigVersion(clusterName, configType);
@@ -348,7 +350,7 @@ public abstract class AbstractProviderModule implements ProviderModule,
                 new String[]{"timeline.metrics.service.webapp.address"}));
 
           if (!configProperties.isEmpty()) {
-            return getPortString(configProperties.get("METRIC_COLLECTOR"));
+            collectorPort = getPortString(configProperties.get("METRIC_COLLECTOR"));
           }
         }
 
@@ -357,9 +359,13 @@ public abstract class AbstractProviderModule implements ProviderModule,
       } catch (UnsupportedPropertyException e) {
         LOG.warn("Failed to retrieve collector port.", e);
       }
-      return "8188";
+      if (collectorPort == null) {
+        return "8188";
+      } else {
+        clusterMetricServerPort = collectorPort;
+      }
     }
-    return null;
+    return clusterMetricServerPort;
   }
 
   @Override

ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-log4j.xml

@@ -119,7 +119,7 @@
 
       log4j.logger.org.apache.zookeeper=INFO
       #log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
-      log4j.logger.org.apache.hadoop.hbase=DEBUG
+      log4j.logger.org.apache.hadoop.hbase=INFO
       # Make these two classes INFO-level. Make them DEBUG to see more zk debug.
       log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO
       log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO

ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-site.xml

@@ -31,6 +31,13 @@
       3) external: External HBase storage backend
     </description>
   </property>
+  <property>
+    <name>timeline.metrics.service.webapp.address</name>
+    <value>0.0.0.0:6188</value>
+    <description>
+      The address of the metrics service web application.
+    </description>
+  </property>
   <property>
     <name>timeline.metrics.aggregator.checkpoint.dir</name>
     <value>/tmp</value>

ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/params.py

@@ -35,7 +35,10 @@ ams_collector_conf_dir = "/etc/ambari-metrics-collector/conf"
 ams_collector_pid_dir = status_params.ams_collector_pid_dir
 ams_collector_hosts = default("/clusterHostInfo/metric_collector_hosts", [])
 ams_collector_host_single = ams_collector_hosts[0] #TODO cardinality is 1+ so we can have more than one host
-ams_collector_port = 8188
+metric_collector_port = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:8188")
+if metric_collector_port and metric_collector_port.find(':') != -1:
+  metric_collector_port = metric_collector_port.split(':')[1]
+pass
 
 
 ams_monitor_conf_dir = "/etc/ambari-metrics-monitor/conf/"

ambari-server/src/main/resources/common-services/AMS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2

@@ -40,17 +40,17 @@ hbase.extendedperiod = 3600
 
 hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
 hbase.period=10
-hbase.collector={{ams_collector_host_single}}:8188
+hbase.collector={{ams_collector_host_single}}:{{metric_collector_port}}
 
 jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
 jvm.period=10
-jvm.collector={{ams_collector_host_single}}:8188
+jvm.collector={{ams_collector_host_single}}:{{metric_collector_port}}
 
 rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
 rpc.period=10
-rpc.collector={{ams_collector_host_single}}:8188
+rpc.collector={{ams_collector_host_single}}:{{metric_collector_port}}
 
 hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
 hbase.sink.timeline.period=10
-hbase.sink.timeline.collector={{ams_collector_host_single}}:8188
+hbase.sink.timeline.collector={{ams_collector_host_single}}:{{metric_collector_port}}
 hbase.sink.timeline.serviceName-prefix=ams

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params.py

@@ -90,6 +90,10 @@ ams_collector_hosts = default("/clusterHostInfo/metric_collector_hosts", [])
 has_metric_collector = not len(ams_collector_hosts) == 0
 if has_metric_collector:
   metric_collector_host = ams_collector_hosts[0]
+  metric_collector_port = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:8188")
+  if metric_collector_port and metric_collector_port.find(':') != -1:
+    metric_collector_port = metric_collector_port.split(':')[1]
+  pass
 
 # if hbase is selected the hbase_rs_hosts, should not be empty, but still default just in case
 if 'slave_hosts' in config['clusterHostInfo']:

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2

@@ -51,19 +51,19 @@ hbase.extendedperiod = 3600
 *.timline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar
 hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
 hbase.period=10
-hbase.collector={{metric_collector_host}}:8188
+hbase.collector={{metric_collector_host}}:{{metric_collector_port}}
 
 jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
 jvm.period=10
-jvm.collector={{metric_collector_host}}:8188
+jvm.collector={{metric_collector_host}}:{{metric_collector_port}}
 
 rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
 rpc.period=10
-rpc.collector={{metric_collector_host}}:8188
+rpc.collector={{metric_collector_host}}:{{metric_collector_port}}
 
 hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
 hbase.sink.timeline.period=10
-hbase.sink.timeline.collector={{metric_collector_host}}:8188
+hbase.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
 
 {% else %}
 

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py

@@ -90,6 +90,10 @@ if has_ganglia_server:
   ganglia_server_host = ganglia_server_hosts[0]
 if has_metric_collector:
   metric_collector_host = ams_collector_hosts[0]
+  metric_collector_port = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:8188")
+  if metric_collector_port and metric_collector_port.find(':') != -1:
+    metric_collector_port = metric_collector_port.split(':')[1]
+  pass
 
 #hadoop params
 

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2

@@ -70,16 +70,17 @@ resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue
 *.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar
 *.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
 *.sink.timeline.period=10
-datanode.sink.timeline.collector={{metric_collector_host}}:8188
-namenode.sink.timeline.collector={{metric_collector_host}}:8188
-resourcemanager.sink.timeline.collector={{metric_collector_host}}:8188
-nodemanager.sink.timeline.collector={{metric_collector_host}}:8188
-historyserver.sink.timeline.collector={{metric_collector_host}}:8188
-journalnode.sink.timeline.collector={{metric_collector_host}}:8188
-nimbus.sink.timeline.collector={{metric_collector_host}}:8188
-supervisor.sink.timeline.collector={{metric_collector_host}}:8188
-maptask.sink.timeline.collector={{metric_collector_host}}:8188
-reducetask.sink.timeline.collector={{metric_collector_host}}:8188
+
+datanode.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
+namenode.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
+resourcemanager.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
+nodemanager.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
+historyserver.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
+journalnode.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
+nimbus.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
+supervisor.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
+maptask.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
+reducetask.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
 
 resourcemanager.sink.timeline.tagsForPrefix.yarn=Queue