AMBARI-9592 AMS: no metrics showing for services (dsen)

ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java

@@ -86,7 +86,7 @@ public class FlumeTimelineMetricsSink extends AbstractTimelineMetricsSink implem
     metricsCache = new TimelineMetricsCache(maxRowCacheSize, metricsSendInterval);
     collectorUri = "http://" + configuration.getProperty(COLLECTOR_HOST_PROPERTY) + "/ws/v1/timeline/metrics";
     List<InetSocketAddress> socketAddresses =
-        Servers.parse(configuration.getProperty(configuration.getProperty(COLLECTOR_HOST_PROPERTY)), 8188);
+        Servers.parse(configuration.getProperty(configuration.getProperty(COLLECTOR_HOST_PROPERTY)), 6188);
     if (socketAddresses != null && !socketAddresses.isEmpty()) {
       socketAddress = socketAddresses.get(0);
     }

ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java

@@ -64,7 +64,7 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple
     serviceName = getServiceName(conf);
 
     // Load collector configs
-    metricsServers = Servers.parse(conf.getString(COLLECTOR_HOST_PROPERTY), 8188);
+    metricsServers = Servers.parse(conf.getString(COLLECTOR_HOST_PROPERTY), 6188);
 
     if (metricsServers == null || metricsServers.isEmpty()) {
       LOG.error("No Metric collector configured.");

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-log4j.xml

@@ -26,116 +26,116 @@
     <name>content</name>
     <description>Custom log4j.properties</description>
     <value>
-      # Licensed to the Apache Software Foundation (ASF) under one
-      # or more contributor license agreements.  See the NOTICE file
-      # distributed with this work for additional information
-      # regarding copyright ownership.  The ASF licenses this file
-      # to you under the Apache License, Version 2.0 (the
-      # "License"); you may not use this file except in compliance
-      # with the License.  You may obtain a copy of the License at
-      #
-      #     http://www.apache.org/licenses/LICENSE-2.0
-      #
-      # Unless required by applicable law or agreed to in writing, software
-      # distributed under the License is distributed on an "AS IS" BASIS,
-      # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-      # See the License for the specific language governing permissions and
-      # limitations under the License.
-
-
-      # Define some default values that can be overridden by system properties
-      hbase.root.logger=INFO,console
-      hbase.security.logger=INFO,console
-      hbase.log.dir=.
-      hbase.log.file=hbase.log
-
-      # Define the root logger to the system property "hbase.root.logger".
-      log4j.rootLogger=${hbase.root.logger}
-
-      # Logging Threshold
-      log4j.threshold=ALL
-
-      #
-      # Daily Rolling File Appender
-      #
-      log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-      log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file}
-
-      # Rollver at midnight
-      log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-      # 30-day backup
-      #log4j.appender.DRFA.MaxBackupIndex=30
-      log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-      # Pattern format: Date LogLevel LoggerName LogMessage
-      log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
-
-      # Rolling File Appender properties
-      hbase.log.maxfilesize=256MB
-      hbase.log.maxbackupindex=20
-
-      # Rolling File Appender
-      log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-      log4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file}
-
-      log4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize}
-      log4j.appender.RFA.MaxBackupIndex=${hbase.log.maxbackupindex}
-
-      log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-      log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
-
-      #
-      # Security audit appender
-      #
-      hbase.security.log.file=SecurityAuth.audit
-      hbase.security.log.maxfilesize=256MB
-      hbase.security.log.maxbackupindex=20
-      log4j.appender.RFAS=org.apache.log4j.RollingFileAppender
-      log4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file}
-      log4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize}
-      log4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex}
-      log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
-      log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-      log4j.category.SecurityLogger=${hbase.security.logger}
-      log4j.additivity.SecurityLogger=false
-      #log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE
-
-      #
-      # Null Appender
-      #
-      log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
-
-      #
-      # console
-      # Add "console" to rootlogger above if you want to use this
-      #
-      log4j.appender.console=org.apache.log4j.ConsoleAppender
-      log4j.appender.console.target=System.err
-      log4j.appender.console.layout=org.apache.log4j.PatternLayout
-      log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
-
-      # Custom Logging levels
-
-      log4j.logger.org.apache.zookeeper=INFO
-      #log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
-      log4j.logger.org.apache.hadoop.hbase=INFO
-      # Make these two classes INFO-level. Make them DEBUG to see more zk debug.
-      log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO
-      log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO
-      #log4j.logger.org.apache.hadoop.dfs=DEBUG
-      # Set this class to log INFO only otherwise its OTT
-      # Enable this to get detailed connection error/retry logging.
-      # log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE
-
-
-      # Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output)
-      #log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG
-
-      # Uncomment the below if you want to remove logging of client region caching'
-      # and scan of .META. messages
-      # log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO
-      # log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Define some default values that can be overridden by system properties
+hbase.root.logger=INFO,console
+hbase.security.logger=INFO,console
+hbase.log.dir=.
+hbase.log.file=hbase.log
+
+# Define the root logger to the system property "hbase.root.logger".
+log4j.rootLogger=${hbase.root.logger}
+
+# Logging Threshold
+log4j.threshold=ALL
+
+#
+# Daily Rolling File Appender
+#
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+# Rolling File Appender properties
+hbase.log.maxfilesize=256MB
+hbase.log.maxbackupindex=20
+
+# Rolling File Appender
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file}
+
+log4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize}
+log4j.appender.RFA.MaxBackupIndex=${hbase.log.maxbackupindex}
+
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+#
+# Security audit appender
+#
+hbase.security.log.file=SecurityAuth.audit
+hbase.security.log.maxfilesize=256MB
+hbase.security.log.maxbackupindex=20
+log4j.appender.RFAS=org.apache.log4j.RollingFileAppender
+log4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file}
+log4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize}
+log4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex}
+log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.category.SecurityLogger=${hbase.security.logger}
+log4j.additivity.SecurityLogger=false
+#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE
+
+#
+# Null Appender
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+# Custom Logging levels
+
+log4j.logger.org.apache.zookeeper=INFO
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+log4j.logger.org.apache.hadoop.hbase=INFO
+# Make these two classes INFO-level. Make them DEBUG to see more zk debug.
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO
+#log4j.logger.org.apache.hadoop.dfs=DEBUG
+# Set this class to log INFO only otherwise its OTT
+# Enable this to get detailed connection error/retry logging.
+# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE
+
+
+# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output)
+#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG
+
+# Uncomment the below if you want to remove logging of client region caching'
+# and scan of .META. messages
+# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO
+# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO
 
     </value>
   </property>

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-log4j.xml

@@ -24,38 +24,38 @@
     <name>content</name>
     <description>Custom log4j.properties</description>
     <value>
-      #
-      # Licensed to the Apache Software Foundation (ASF) under one
-      # or more contributor license agreements.  See the NOTICE file
-      # distributed with this work for additional information
-      # regarding copyright ownership.  The ASF licenses this file
-      # to you under the Apache License, Version 2.0 (the
-      # "License"); you may not use this file except in compliance
-      # with the License.  You may obtain a copy of the License at
-      #
-      #     http://www.apache.org/licenses/LICENSE-2.0
-      #
-      # Unless required by applicable law or agreed to in writing, software
-      # distributed under the License is distributed on an "AS IS" BASIS,
-      # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-      # See the License for the specific language governing permissions and
-      # limitations under the License.
-      #
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 
-      # Define some default values that can be overridden by system properties
-      ams.log.dir=.
-      ams.log.file=ambari-metrics-collector.log
+# Define some default values that can be overridden by system properties
+ams.log.dir=.
+ams.log.file=ambari-metrics-collector.log
 
-      # Root logger option
-      log4j.rootLogger=INFO,file
+# Root logger option
+log4j.rootLogger=INFO,file
 
-      # Direct log messages to a log file
-      log4j.appender.file=org.apache.log4j.RollingFileAppender
-      log4j.appender.file.File=${ams.log.dir}/${ams.log.file}
-      log4j.appender.file.MaxFileSize=80MB
-      log4j.appender.file.MaxBackupIndex=60
-      log4j.appender.file.layout=org.apache.log4j.PatternLayout
-      log4j.appender.file.layout.ConversionPattern=%d{ABSOLUTE} %5p [%t] %c{1}:%L - %m%n
+# Direct log messages to a log file
+log4j.appender.file=org.apache.log4j.RollingFileAppender
+log4j.appender.file.File=${ams.log.dir}/${ams.log.file}
+log4j.appender.file.MaxFileSize=80MB
+log4j.appender.file.MaxBackupIndex=60
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d{ABSOLUTE} %5p [%t] %c{1}:%L - %m%n
     </value>
   </property>
 

ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py

@@ -80,7 +80,7 @@ hostname = None
 if config.has_key('hostname'):
   hostname = config['hostname']
 
-ams_collector_hosts = default("/clusterHostInfo/metric_collector_hosts", [])
+ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
 has_metric_collector = not len(ams_collector_hosts) == 0
 if has_metric_collector:
   metric_collector_host = ams_collector_hosts[0]

ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2

@@ -16,7 +16,7 @@
 # limitations under the License.
 #}
 
-collector={{metric_collector_host}}:8188
+collector={{metric_collector_host}}:6188
 collectionFrequency=60000
 maxRowCacheSize=10000
 sendInterval=59000

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params.py

@@ -89,11 +89,11 @@ regionserver_jaas_config_file = format("{hbase_conf_dir}/hbase_regionserver_jaas
 ganglia_server_hosts = default('/clusterHostInfo/ganglia_server_host', []) # is not passed when ganglia is not present
 ganglia_server_host = '' if len(ganglia_server_hosts) == 0 else ganglia_server_hosts[0]
 
-ams_collector_hosts = default("/clusterHostInfo/metric_collector_hosts", [])
+ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
 has_metric_collector = not len(ams_collector_hosts) == 0
 if has_metric_collector:
   metric_collector_host = ams_collector_hosts[0]
-  metric_collector_port = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:8188")
+  metric_collector_port = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
   if metric_collector_port and metric_collector_port.find(':') != -1:
     metric_collector_port = metric_collector_port.split(':')[1]
   pass

ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py

@@ -76,12 +76,12 @@ metric_collector_port = ""
 if ganglia_installed:
   kafka_metrics_reporters = "kafka.ganglia.KafkaGangliaMetricsReporter"
 
-ams_collector_hosts = default("/clusterHostInfo/metric_collector_hosts", [])
+ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
 has_metric_collector = not len(ams_collector_hosts) == 0
 
 if has_metric_collector:
   metric_collector_host = ams_collector_hosts[0]
-  metric_collector_port = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:8188")
+  metric_collector_port = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
   if metric_collector_port and metric_collector_port.find(':') != -1:
     metric_collector_port = metric_collector_port.split(':')[1]
 

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py

@@ -93,7 +93,7 @@ if security_enabled:
     nimbus_bare_jaas_principal = _nimbus_principal_name.replace('/_HOST','').replace('@'+kerberos_domain,'')
     nimbus_keytab_path = config['configurations']['storm-env']['nimbus_keytab']
 
-ams_collector_hosts = default("/clusterHostInfo/metric_collector_hosts", [])
+ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
 has_metric_collector = not len(ams_collector_hosts) == 0
 if has_metric_collector:
   metric_collector_host = ams_collector_hosts[0]

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py

@@ -67,7 +67,7 @@ jtnode_host = default("/clusterHostInfo/jtnode_host", [])
 namenode_host = default("/clusterHostInfo/namenode_host", [])
 zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
 ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", [])
-ams_collector_hosts = default("/clusterHostInfo/metric_collector_hosts", [])
+ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
 
 has_namenode = not len(namenode_host) == 0
 has_resourcemanager = not len(rm_host) == 0
@@ -90,7 +90,7 @@ if has_ganglia_server:
   ganglia_server_host = ganglia_server_hosts[0]
 if has_metric_collector:
   metric_collector_host = ams_collector_hosts[0]
-  metric_collector_port = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:8188")
+  metric_collector_port = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
   if metric_collector_port and metric_collector_port.find(':') != -1:
     metric_collector_port = metric_collector_port.split(':')[1]
   pass

ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/before-START/scripts/params.py

@@ -20,11 +20,11 @@ limitations under the License.
 from resource_management import *
 import nturl2path
 config = Script.get_config()
-ams_collector_hosts = default("/clusterHostInfo/metric_collector_hosts", [])
+ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
 has_metric_collector = not len(ams_collector_hosts) == 0
 if has_metric_collector:
   metric_collector_host = ams_collector_hosts[0]
-  metric_collector_port = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:8188")
+  metric_collector_port = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
   if metric_collector_port and metric_collector_port.find(':') != -1:
     metric_collector_port = metric_collector_port.split(':')[1]
   pass

ambari-server/src/test/python/stacks/2.2/configs/journalnode-upgrade-hdfs-secure.json

@@ -1246,7 +1246,7 @@
         "slave_hosts": [
             "c6406.ambari.apache.org"
         ], 
-        "metric_monitor_hosts": [
+        "metrics_monitor_hosts": [
             "c6408.ambari.apache.org", 
             "c6407.ambari.apache.org", 
             "c6406.ambari.apache.org", 
@@ -1264,7 +1264,7 @@
         "webhcat_server_host": [
             "c6407.ambari.apache.org"
         ], 
-        "metric_collector_hosts": [
+        "metrics_collector_hosts": [
             "c6408.ambari.apache.org"
         ], 
         "ambari_server_host": [

ambari-server/src/test/python/stacks/2.2/configs/journalnode-upgrade.json

@@ -1246,7 +1246,7 @@
         "slave_hosts": [
             "c6406.ambari.apache.org"
         ], 
-        "metric_monitor_hosts": [
+        "metrics_monitor_hosts": [
             "c6408.ambari.apache.org", 
             "c6407.ambari.apache.org", 
             "c6406.ambari.apache.org", 
@@ -1264,7 +1264,7 @@
         "webhcat_server_host": [
             "c6407.ambari.apache.org"
         ], 
-        "metric_collector_hosts": [
+        "metrics_collector_hosts": [
             "c6408.ambari.apache.org"
         ], 
         "ambari_server_host": [