AMBARI-19141 : Refinements for AMS whitelisting. (avijayan)

ambari-metrics/ambari-metrics-assembly/pom.xml

@@ -334,6 +334,9 @@
                         <source>
                           <location>${collector.dir}/conf/unix/log4j.properties</location>
                         </source>
+                        <source>
+                          <location>${collector.dir}/conf/unix/metrics_whitelist</location>
+                        </source>
                         <source>
                           <location>${collector.dir}/target/embedded/${hbase.folder}/conf/hbase-site.xml</location>
                         </source>
@@ -869,6 +872,15 @@
                     <prefix>/etc/ambari-metrics-collector/conf</prefix>
                   </mapper>
                 </data>
+                <data>
+                  <src>${collector.dir}/conf/unix/metrics_whitelist</src>
+                  <type>file</type>
+                  <mapper>
+                    <type>perm</type>
+                    <filemode>755</filemode>
+                    <prefix>/etc/ambari-metrics-collector/conf</prefix>
+                  </mapper>
+                </data>
                 <data>
                   <src>${collector.dir}/target/embedded/${hbase.folder}/conf/hbase-site.xml</src>
                   <type>file</type>

ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml

@@ -45,6 +45,7 @@
         <include>ams-env.cmd</include>
         <include>ams-site.xml</include>
         <include>log4j.properties</include>
+        <include>metrics_whitelist</include>
       </includes>
     </fileSet>
     <fileSet>

ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector.xml

@@ -44,6 +44,7 @@
         <include>ams-env.sh</include>
         <include>ams-site.xml</include>
         <include>log4j.properties</include>
+        <include>metrics_whitelist</include>
       </includes>
     </fileSet>
     <fileSet>

ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricMetadata.java

@@ -36,6 +36,7 @@ public class TimelineMetricMetadata {
   private String type = "UNDEFINED";
   private Long seriesStartTime;
   boolean supportsAggregates = true;
+  boolean isWhitelisted = false;
   // Serialization ignored helper flag
   boolean isPersisted = false;
 
@@ -52,13 +53,14 @@ public class TimelineMetricMetadata {
 
   public TimelineMetricMetadata(String metricName, String appId, String units,
                                 String type, Long seriesStartTime,
-                                boolean supportsAggregates) {
+                                boolean supportsAggregates, boolean isWhitelisted) {
     this.metricName = metricName;
     this.appId = appId;
     this.units = units;
     this.type = type;
     this.seriesStartTime = seriesStartTime;
     this.supportsAggregates = supportsAggregates;
+    this.isWhitelisted = isWhitelisted;
   }
 
   @XmlElement(name = "metricname")
@@ -112,6 +114,11 @@ public class TimelineMetricMetadata {
     return supportsAggregates;
   }
 
+  @XmlElement(name = "isWhitelisted")
+  public boolean isWhitelisted() {
+    return isWhitelisted;
+  }
+
   public void setSupportsAggregates(boolean supportsAggregates) {
     this.supportsAggregates = supportsAggregates;
   }
@@ -140,7 +147,8 @@ public class TimelineMetricMetadata {
     return (this.units != null && !this.units.equals(metadata.getUnits())) ||
       (this.type != null && !this.type.equals(metadata.getType())) ||
       //!this.lastRecordedTime.equals(metadata.getLastRecordedTime()) || // TODO: support
-      !this.supportsAggregates == metadata.isSupportsAggregates();
+      !this.supportsAggregates == metadata.isSupportsAggregates() ||
+      this.isWhitelisted != metadata.isWhitelisted;
   }
 
   @Override

ambari-metrics/ambari-metrics-timelineservice/conf/unix/metrics_whitelist

@@ -0,0 +1,527 @@
+AMBARI_METRICS.SmokeTest.FakeMetric
+ActiveThreads
+BytesQueued
+BytesReceivedLast5Minutes
+BytesSentLast5Minutes
+ChannelSize
+EventPutSuccessCount
+EventTakeSuccessCount
+FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Capacity
+FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.DfsUsed
+FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.NumFailedVolumes
+FlowFilesQueued
+FlowFilesReceivedLast5Minutes
+FlowFilesSentLast5Minutes
+Free Slots
+Supervisors
+TimelineMetricStoreWatcher.FakeMetric
+Topologies
+Total Executors
+Total Slots
+Total Tasks
+Used Slots
+bytes_in
+bytes_out
+cache.CacheMetrics.CacheCapacityRemaining
+cache.CacheMetrics.CacheCapacityRemainingPercentage
+cache.CacheMetrics.CacheCapacityTotal
+cache.CacheMetrics.CacheCapacityUsed
+cache.CacheMetrics.CacheHitRatio
+cache.CacheMetrics.CacheReadRequests
+cpu_idle
+cpu_intr
+cpu_nice
+cpu_num
+cpu_sintr
+cpu_steal
+cpu_system
+cpu_user
+cpu_wio
+default.General.active_calls_api_hs2_operation_INITIALIZED
+default.General.active_calls_api_hs2_operation_PENDING
+default.General.active_calls_api_hs2_operation_RUNNING
+default.General.api_create_table_count
+default.General.api_get_all_databases_75thpercentile
+default.General.api_get_all_databases_999thpercentile
+default.General.api_get_partitions_by_names_75thpercentile
+default.General.api_get_partitions_by_names_999thpercentile
+default.General.api_hs2_operation_INITIALIZED_count
+default.General.create_total_count_partitions
+default.General.hs2_completed_operation_CLOSED
+default.General.hs2_completed_operation_ERROR
+default.General.hs2_completed_operation_FINISHED
+default.General.init_total_count_dbs
+default.General.init_total_count_partitions
+default.General.init_total_count_tables
+default.General.jvm.pause.extraSleepTime
+default.General.memory.heap.committed
+default.General.memory.heap.max
+default.General.memory.heap.used
+default.General.memory.non-heap.committed
+default.General.memory.non-heap.max
+default.General.memory.non-heap.used
+dfs.FSNamesystem.CapacityTotalGB
+dfs.FSNamesystem.CapacityUsedGB
+dfs.FSNamesystem.ExpiredHeartbeats
+dfs.FSNamesystem.LastCheckpointTime
+dfs.FSNamesystem.LastWrittenTransactionId
+dfs.FSNamesystem.LockQueueLength
+dfs.FSNamesystem.MillisSinceLastLoadedEdits
+dfs.FSNamesystem.MissingReplOneBlocks
+dfs.FSNamesystem.NumFilesUnderConstruction
+dfs.FSNamesystem.PendingDataNodeMessageCount
+dfs.FSNamesystem.PostponedMisreplicatedBlocks
+dfs.FSNamesystem.Snapshots
+dfs.FSNamesystem.SnapshottableDirectories
+dfs.FSNamesystem.StaleDataNodes
+dfs.FSNamesystem.TotalFiles
+dfs.FSNamesystem.TotalLoad
+dfs.FSNamesystem.TransactionsSinceLastCheckpoint
+dfs.FSNamesystem.TransactionsSinceLastLogRoll
+dfs.FSNamesystem.UnderReplicatedBlocks
+dfs.datanode.BlocksRead
+dfs.datanode.BlocksWritten
+dfs.datanode.DatanodeNetworkErrors
+dfs.datanode.FsyncNanosAvgTime
+dfs.datanode.FsyncNanosNumOps
+dfs.datanode.PacketAckRoundTripTimeNanosAvgTime
+dfs.datanode.PacketAckRoundTripTimeNanosNumOps
+dfs.datanode.SendDataPacketBlockedOnNetworkNanosAvgTime
+dfs.datanode.SendDataPacketBlockedOnNetworkNanosNumOps
+dfs.datanode.SendDataPacketTransferNanosAvgTime
+dfs.datanode.SendDataPacketTransferNanosNumOps
+dfs.namenode.TotalFileOps
+disk_free
+disk_total
+disk_used
+executors.ExecutorMetrics.ExecutorAvailableFreeSlots
+executors.ExecutorMetrics.ExecutorAvailableFreeSlotsPercent
+executors.ExecutorMetrics.ExecutorCacheMemoryPerInstance
+executors.ExecutorMetrics.ExecutorJvmMaxMemory
+executors.ExecutorMetrics.ExecutorMaxFreeSlots
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost_300s50thPercentileLatency
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost_300s90thPercentileLatency
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost_300s95thPercentileLatency
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost_300s99thPercentileLatency
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill_300s50thPercentileLatency
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill_300s90thPercentileLatency
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill_300s95thPercentileLatency
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill_300s99thPercentileLatency
+executors.ExecutorMetrics.ExecutorMemoryPerInstance
+executors.ExecutorMetrics.ExecutorNumExecutorsPerInstance
+executors.ExecutorMetrics.ExecutorNumQueuedRequests
+executors.ExecutorMetrics.ExecutorTotalEvictedFromWaitQueue
+executors.ExecutorMetrics.ExecutorTotalFailed
+executors.ExecutorMetrics.ExecutorTotalKilled
+executors.ExecutorMetrics.ExecutorTotalRejectedRequests
+executors.ExecutorMetrics.ExecutorTotalRequestsHandled
+executors.ExecutorMetrics.ExecutorTotalSuccess
+gc.ConcurrentMarkSweep.count
+gc.ConcurrentMarkSweep.time
+gc.ParNew.count
+gc.ParNew.time
+io.IOMetrics.MaxDecodingTime
+io.IOMetrics.PercentileDecodingTime_30s50thPercentileLatency
+io.IOMetrics.PercentileDecodingTime_30s90thPercentileLatency
+io.IOMetrics.PercentileDecodingTime_30s95thPercentileLatency
+io.IOMetrics.PercentileDecodingTime_30s99thPercentileLatency
+ipc.client.org.apache.hadoop.ipc.DecayRpcScheduler.Caller(*).Priority
+ipc.client.org.apache.hadoop.ipc.DecayRpcScheduler.Caller(*).Volume
+jvm.JvmMetrics.GcCount
+jvm.JvmMetrics.GcCountConcurrentMarkSweep
+jvm.JvmMetrics.GcCountParNew
+jvm.JvmMetrics.GcNumWarnThresholdExceeded
+jvm.JvmMetrics.GcTimeMillis
+jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep
+jvm.JvmMetrics.GcTimeMillisParNew
+jvm.JvmMetrics.GcTotalExtraSleepTime
+jvm.JvmMetrics.LogError
+jvm.JvmMetrics.LogFatal
+jvm.JvmMetrics.MemHeapCommittedM
+jvm.JvmMetrics.MemHeapMaxM
+jvm.JvmMetrics.MemHeapUsedM
+jvm.JvmMetrics.MemNonHeapCommittedM
+jvm.JvmMetrics.MemNonHeapMaxM
+jvm.JvmMetrics.MemNonHeapUsedM
+jvm.JvmMetrics.ThreadsBlocked
+jvm.JvmMetrics.ThreadsNew
+jvm.JvmMetrics.ThreadsRunnable
+jvm.JvmMetrics.ThreadsTerminated
+jvm.JvmMetrics.ThreadsTimedWaiting
+jvm.JvmMetrics.ThreadsWaiting
+jvm.Master.JvmMetrics.ThreadsBlocked
+jvm.Master.JvmMetrics.ThreadsNew
+jvm.Master.JvmMetrics.ThreadsRunnable
+jvm.Master.JvmMetrics.ThreadsTerminated
+jvm.Master.JvmMetrics.ThreadsTimedWaiting
+jvm.Master.JvmMetrics.ThreadsWaiting
+jvm.RegionServer.JvmMetrics.GcCount
+jvm.RegionServer.JvmMetrics.GcCountConcurrentMarkSweep
+jvm.RegionServer.JvmMetrics.GcCountParNew
+jvm.RegionServer.JvmMetrics.GcTimeMillis
+jvm.RegionServer.JvmMetrics.GcTimeMillisConcurrentMarkSweep
+jvm.RegionServer.JvmMetrics.GcTimeMillisParNew
+jvm.RegionServer.JvmMetrics.MemHeapCommittedM
+jvm.RegionServer.JvmMetrics.MemHeapMaxM
+jvm.RegionServer.JvmMetrics.MemHeapUsedM
+jvm.RegionServer.JvmMetrics.MemNonHeapCommittedM
+jvm.RegionServer.JvmMetrics.MemNonHeapMaxM
+jvm.RegionServer.JvmMetrics.MemNonHeapUsedM
+jvm.RegionServer.JvmMetrics.ThreadsBlocked
+jvm.RegionServer.JvmMetrics.ThreadsNew
+jvm.RegionServer.JvmMetrics.ThreadsRunnable
+jvm.RegionServer.JvmMetrics.ThreadsTerminated
+jvm.RegionServer.JvmMetrics.ThreadsTimedWaiting
+jvm.RegionServer.JvmMetrics.ThreadsWaiting
+jvm.daemon_thread_count
+jvm.file_descriptor_usage
+jvm.heap_usage
+jvm.thread_count
+jvm.uptime
+kafka.controller.ControllerStats.LeaderElectionRateAndTimeMs.1MinuteRate
+kafka.controller.ControllerStats.LeaderElectionRateAndTimeMs.count
+kafka.controller.ControllerStats.UncleanLeaderElectionsPerSec.1MinuteRate
+kafka.controller.ControllerStats.UncleanLeaderElectionsPerSec.count
+kafka.controller.KafkaController.ActiveControllerCount
+kafka.controller.KafkaController.OfflinePartitionsCount
+kafka.log.LogFlushStats.LogFlushRateAndTimeMs.1MinuteRate
+kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.count
+kafka.network.RequestMetrics.RequestsPerSec.request.Produce.count
+kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.99percentile
+kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.max
+kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.mean
+kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.min
+kafka.network.RequestMetrics.TotalTimeMs.request.FetchFollower.99percentile
+kafka.network.RequestMetrics.TotalTimeMs.request.FetchFollower.max
+kafka.network.RequestMetrics.TotalTimeMs.request.FetchFollower.mean
+kafka.network.RequestMetrics.TotalTimeMs.request.FetchFollower.min
+kafka.network.RequestMetrics.TotalTimeMs.request.Produce.99percentile
+kafka.network.RequestMetrics.TotalTimeMs.request.Produce.max
+kafka.network.RequestMetrics.TotalTimeMs.request.Produce.mean
+kafka.network.RequestMetrics.TotalTimeMs.request.Produce.min
+kafka.server.BrokerTopicMetrics.BytesInPerSec.1MinuteRate
+kafka.server.BrokerTopicMetrics.BytesInPerSec.count
+kafka.server.BrokerTopicMetrics.BytesInPerSec.topic.*.count
+kafka.server.BrokerTopicMetrics.BytesOutPerSec.1MinuteRate
+kafka.server.BrokerTopicMetrics.BytesOutPerSec.count
+kafka.server.BrokerTopicMetrics.BytesOutPerSec.topic.*.count
+kafka.server.BrokerTopicMetrics.MessagesInPerSec.1MinuteRate
+kafka.server.BrokerTopicMetrics.MessagesInPerSec.count
+kafka.server.BrokerTopicMetrics.MessagesInPerSec.topic.*.count
+kafka.server.BrokerTopicMetrics.TotalProduceRequestsPerSec.topic.*.count
+kafka.server.FetcherLagMetrics.ConsumerLag.clientId.ReplicaFetcherThread-0-1001.partition.0.topic.*
+kafka.server.ReplicaFetcherManager.MaxLag.clientId.Replica
+kafka.server.ReplicaManager.IsrExpandsPerSec.count
+kafka.server.ReplicaManager.IsrShrinksPerSec.count
+kafka.server.ReplicaManager.LeaderCount
+kafka.server.ReplicaManager.PartitionCount
+kafka.server.ReplicaManager.UnderReplicatedPartitions
+load_fifteen
+load_five
+load_one
+master.AssignmentManger.ritCount
+master.AssignmentManger.ritCountOverThreshold
+master.AssignmentManger.ritOldestAge
+master.Master.QueueCallTime_median
+master.Master.TotalCallTime_num_ops
+master.Master.exceptions
+master.Server.averageLoad
+master.Server.numDeadRegionServers
+master.Server.numRegionServers
+mem_buffered
+mem_cached
+mem_free
+mem_shared
+mem_total
+mem_used
+memory.heap.committed
+memory.heap.max
+memory.heap.used
+memory.non-heap.committed
+memory.non-heap.max
+memory.non-heap.used
+pkts_in
+pkts_out
+proc_run
+proc_total
+read_bps
+read_bytes
+read_count
+read_time
+regionserver.RegionServer.ProcessCallTime_75th_percentile
+regionserver.RegionServer.ProcessCallTime_95th_percentile
+regionserver.RegionServer.ProcessCallTime_99th_percentile
+regionserver.RegionServer.ProcessCallTime_max
+regionserver.RegionServer.ProcessCallTime_mean
+regionserver.RegionServer.ProcessCallTime_median
+regionserver.RegionServer.QueueCallTime_75th_percentile
+regionserver.RegionServer.QueueCallTime_95th_percentile
+regionserver.RegionServer.QueueCallTime_99th_percentile
+regionserver.RegionServer.QueueCallTime_max
+regionserver.RegionServer.QueueCallTime_mean
+regionserver.RegionServer.QueueCallTime_median
+regionserver.RegionServer.TotalCallTime_num_ops
+regionserver.RegionServer.authenticationFailures
+regionserver.RegionServer.authenticationSuccesses
+regionserver.RegionServer.authorizationFailures
+regionserver.RegionServer.authorizationSuccesses
+regionserver.RegionServer.exceptions
+regionserver.RegionServer.numActiveHandler
+regionserver.RegionServer.numCallsInGeneralQueue
+regionserver.RegionServer.numCallsInPriorityQueue
+regionserver.RegionServer.numCallsInReplicationQueue
+regionserver.RegionServer.numOpenConnections
+regionserver.RegionServer.queueSize
+regionserver.RegionServer.receivedBytes
+regionserver.RegionServer.sentBytes
+regionserver.Server.Append_75th_percentile
+regionserver.Server.Append_95th_percentile
+regionserver.Server.Append_99th_percentile
+regionserver.Server.Append_max
+regionserver.Server.Append_mean
+regionserver.Server.Append_median
+regionserver.Server.Append_min
+regionserver.Server.Append_num_ops
+regionserver.Server.Delete_75th_percentile
+regionserver.Server.Delete_95th_percentile
+regionserver.Server.Delete_99th_percentile
+regionserver.Server.Delete_max
+regionserver.Server.Delete_mean
+regionserver.Server.Delete_median
+regionserver.Server.Delete_min
+regionserver.Server.Delete_num_ops
+regionserver.Server.Get_75th_percentile
+regionserver.Server.Get_95th_percentile
+regionserver.Server.Get_99th_percentile
+regionserver.Server.Get_max
+regionserver.Server.Get_mean
+regionserver.Server.Get_median
+regionserver.Server.Get_min
+regionserver.Server.Get_num_ops
+regionserver.Server.Increment_75th_percentile
+regionserver.Server.Increment_95th_percentile
+regionserver.Server.Increment_99th_percentile
+regionserver.Server.Increment_max
+regionserver.Server.Increment_mean
+regionserver.Server.Increment_median
+regionserver.Server.Increment_min
+regionserver.Server.Increment_num_ops
+regionserver.Server.Mutate_75th_percentile
+regionserver.Server.Mutate_95th_percentile
+regionserver.Server.Mutate_99th_percentile
+regionserver.Server.Mutate_max
+regionserver.Server.Mutate_mean
+regionserver.Server.Mutate_median
+regionserver.Server.Mutate_min
+regionserver.Server.Mutate_num_ops
+regionserver.Server.Replay_75th_percentile
+regionserver.Server.Replay_95th_percentile
+regionserver.Server.Replay_99th_percentile
+regionserver.Server.Replay_max
+regionserver.Server.Replay_mean
+regionserver.Server.Replay_median
+regionserver.Server.Replay_min
+regionserver.Server.Replay_num_ops
+regionserver.Server.ScanNext_95th_percentile
+regionserver.Server.ScanNext_num_ops
+regionserver.Server.ScanTime_75th_percentile
+regionserver.Server.ScanTime_95th_percentile
+regionserver.Server.ScanTime_99th_percentile
+regionserver.Server.ScanTime_max
+regionserver.Server.ScanTime_mean
+regionserver.Server.ScanTime_median
+regionserver.Server.blockCacheCount
+regionserver.Server.blockCacheCountHitPercent
+regionserver.Server.blockCacheEvictionCount
+regionserver.Server.blockCacheExpressHitPercent
+regionserver.Server.blockCacheFreeSize
+regionserver.Server.blockCacheHitCount
+regionserver.Server.blockCacheMissCount
+regionserver.Server.blockCacheSize
+regionserver.Server.blockCountHitPercent
+regionserver.Server.compactionQueueLength
+regionserver.Server.flushQueueLength
+regionserver.Server.hlogFileCount
+regionserver.Server.hlogFileSize
+regionserver.Server.memStoreSize
+regionserver.Server.percentFilesLocal
+regionserver.Server.readRequestCount
+regionserver.Server.regionCount
+regionserver.Server.slowDeleteCount
+regionserver.Server.slowGetCount
+regionserver.Server.slowPutCount
+regionserver.Server.splitQueueLength
+regionserver.Server.staticBloomSize
+regionserver.Server.staticIndexSize
+regionserver.Server.storeCount
+regionserver.Server.storeFileCount
+regionserver.Server.storeFileIndexSize
+regionserver.Server.storeFileSize
+regionserver.Server.totalRequestCount
+regionserver.Server.updatesBlockedTime
+regionserver.Server.writeRequestCount
+regionserver.Tables.*_metric_averageRegionSize
+regionserver.Tables.*_metric_avgStoreFileAge
+regionserver.Tables.*_metric_flushMemstoreSize_95th_percentile
+regionserver.Tables.*_metric_flushMemstoreSize_max
+regionserver.Tables.*_metric_flushMemstoreSize_mean
+regionserver.Tables.*_metric_flushMemstoreSize_median
+regionserver.Tables.*_metric_flushOutputSize_95th_percentile
+regionserver.Tables.*_metric_flushOutputSize_max
+regionserver.Tables.*_metric_flushOutputSize_mean
+regionserver.Tables.*_metric_flushOutputSize_median
+regionserver.Tables.*_metric_flushTime_95th_percentile
+regionserver.Tables.*_metric_flushTime_max
+regionserver.Tables.*_metric_flushTime_mean
+regionserver.Tables.*_metric_flushTime_median
+regionserver.Tables.*_metric_flushTime_num_ops
+regionserver.Tables.*_metric_flushedMemstoreBytes
+regionserver.Tables.*_metric_flushedOutputBytes
+regionserver.Tables.*_metric_maxStoreFileAge
+regionserver.Tables.*_metric_memStoreSize
+regionserver.Tables.*_metric_minStoreFileAge
+regionserver.Tables.*_metric_numReferenceFiles
+regionserver.Tables.*_metric_readRequestCount
+regionserver.Tables.*_metric_regionCount
+regionserver.Tables.*_metric_storeCount
+regionserver.Tables.*_metric_storeFileCount
+regionserver.Tables.*_metric_storeFileSize
+regionserver.Tables.*_metric_tableSize
+regionserver.Tables.*_metric_totalRequestCount
+regionserver.Tables.*_metric_writeRequestCount
+regionserver.Users.*_metric_append_num_ops
+regionserver.Users.*_metric_delete_num_ops
+regionserver.Users.*_metric_get_num_ops
+regionserver.Users.*_metric_increment_num_ops
+regionserver.Users.*_metric_mutate_num_ops
+regionserver.Users.*_metric_scanTime_num_ops
+regionserver.WAL.AppendSize_75th_percentile
+regionserver.WAL.AppendSize_95th_percentile
+regionserver.WAL.AppendSize_99th_percentile
+regionserver.WAL.AppendSize_max
+regionserver.WAL.AppendSize_mean
+regionserver.WAL.AppendSize_median
+regionserver.WAL.AppendTime_75th_percentile
+regionserver.WAL.AppendTime_95th_percentile
+regionserver.WAL.AppendTime_99th_percentile
+regionserver.WAL.AppendTime_max
+regionserver.WAL.AppendTime_mean
+regionserver.WAL.AppendTime_median
+regionserver.WAL.SyncTime_75th_percentile
+regionserver.WAL.SyncTime_95th_percentile
+regionserver.WAL.SyncTime_99th_percentile
+regionserver.WAL.SyncTime_max
+regionserver.WAL.SyncTime_mean
+regionserver.WAL.SyncTime_median
+regionserver.WAL.SyncTime_num_ops
+regionserver.WAL.appendCount
+regionserver.WAL.slowAppendCount
+rpc.rpc.CallQueueLength
+rpc.rpc.NumOpenConnections
+rpc.rpc.RpcAuthenticationFailures
+rpc.rpc.RpcAuthorizationFailures
+rpc.rpc.RpcProcessingTimeAvgTime
+rpc.rpc.RpcQueueTimeAvgTime
+rpc.rpc.RpcSlowCalls
+rpc.rpc.client.CallQueueLength
+rpc.rpc.client.NumOpenConnections
+rpc.rpc.client.RpcClientBackoff
+rpc.rpc.client.RpcProcessingTimeAvgTime
+rpc.rpc.client.RpcProcessingTimeNumOps
+rpc.rpc.client.RpcQueueTimeAvgTime
+rpc.rpc.client.RpcQueueTimeNumOps
+rpc.rpc.client.RpcSlowCalls
+rpc.rpc.datanode.CallQueueLength
+rpc.rpc.datanode.RpcProcessingTimeAvgTime
+rpc.rpc.datanode.RpcProcessingTimeNumOps
+rpc.rpc.datanode.RpcQueueTimeAvgTime
+rpc.rpc.datanode.RpcQueueTimeNumOps
+rpc.rpc.datanode.RpcSlowCalls
+rpcdetailed.rpcdetailed.client.AddBlockAvgTime
+rpcdetailed.rpcdetailed.client.AddBlockNumOps
+swap_free
+swap_total
+threads.blocked.count
+threads.count
+threads.daemon.count
+threads.deadlock.count
+threads.runnable.count
+topology.*.%.--ack-count.%
+topology.*.%.--complete-latency.%
+topology.*.%.--emit-count.%
+topology.*.%.--execute-count.%
+topology.*.%.--fail-count.%
+topology.*.%.--receive.population
+topology.*.%.--sendqueue.population
+topology.*.*.%.--ack-count.%
+topology.*.*.%.--complete-latency.%
+topology.*.*.%.--emit-count.%
+topology.*.*.%.--execute-count.%
+topology.*.*.%.--fail-count.%
+topology.*.*.%.--process-latency.%
+topology.*.*.%.--receive.population
+topology.*.*.%.--sendqueue.population
+topology.*.--system.%.%.-1.GC/PSMarkSweep.count
+topology.*.--system.%.%.-1.GC/PSMarkSweep.timeMs
+topology.*.--system.%.%.-1.GC/PSScavenge.count
+topology.*.--system.%.%.-1.GC/PSScavenge.timeMs
+topology.*.--system.%.%.-1.memory/heap.usedBytes
+topology.*.--system.%.%.-1.memory/nonHeap.usedBytes
+topology.*.kafka-topic.*.*.latestCompletedOffset
+topology.*.kafka-topic.*.*.latestTimeOffset
+write_bps
+write_bytes
+write_count
+write_time
+yarn.ClusterMetrics.AMLaunchDelayAvgTime
+yarn.ClusterMetrics.AMRegisterDelayAvgTime
+yarn.ClusterMetrics.NumActiveNMs
+yarn.ClusterMetrics.NumLostNMs
+yarn.ClusterMetrics.NumUnhealthyNMs
+yarn.NodeManagerMetrics.AllocatedContainers
+yarn.NodeManagerMetrics.AllocatedGB
+yarn.NodeManagerMetrics.AllocatedVCores
+yarn.NodeManagerMetrics.AvailableGB
+yarn.NodeManagerMetrics.BadLocalDirs
+yarn.NodeManagerMetrics.BadLogDirs
+yarn.NodeManagerMetrics.ContainerLaunchDurationAvgTime
+yarn.NodeManagerMetrics.ContainersCompleted
+yarn.NodeManagerMetrics.ContainersFailed
+yarn.NodeManagerMetrics.ContainersIniting
+yarn.NodeManagerMetrics.ContainersKilled
+yarn.NodeManagerMetrics.ContainersLaunched
+yarn.NodeManagerMetrics.ContainersRunning
+yarn.NodeManagerMetrics.GoodLocalDirsDiskUtilizationPerc
+yarn.NodeManagerMetrics.GoodLogDirsDiskUtilizationPerc
+yarn.QueueMetrics.Queue=root.AMResourceLimitMB
+yarn.QueueMetrics.Queue=root.AggregateContainersAllocated
+yarn.QueueMetrics.Queue=root.AggregateNodeLocalContainersAllocated
+yarn.QueueMetrics.Queue=root.AggregateOffSwitchContainersAllocated
+yarn.QueueMetrics.Queue=root.AggregateRackLocalContainersAllocated
+yarn.QueueMetrics.Queue=root.AllocatedContainers
+yarn.QueueMetrics.Queue=root.AllocatedMB
+yarn.QueueMetrics.Queue=root.AllocatedVCores
+yarn.QueueMetrics.Queue=root.AppAttemptFirstContainerAllocationDelayAvgTime
+yarn.QueueMetrics.Queue=root.AppsCompleted
+yarn.QueueMetrics.Queue=root.AppsFailed
+yarn.QueueMetrics.Queue=root.AppsKilled
+yarn.QueueMetrics.Queue=root.AppsPending
+yarn.QueueMetrics.Queue=root.AppsRunning
+yarn.QueueMetrics.Queue=root.AppsSubmitted
+yarn.QueueMetrics.Queue=root.AvailableMB
+yarn.QueueMetrics.Queue=root.AvailableVCores
+yarn.QueueMetrics.Queue=root.PendingContainers
+yarn.QueueMetrics.Queue=root.PendingMB
+yarn.QueueMetrics.Queue=root.UsedAMResourceMB
+yarn.QueueMetrics.Queue=root.default.ReservedContainers
+yarn.QueueMetrics.Queue=root.default.ReservedMB
+yarn.QueueMetrics.Queue=root.running_0
+yarn.QueueMetrics.Queue=root.running_1440
+yarn.QueueMetrics.Queue=root.running_300
+yarn.QueueMetrics.Queue=root.running_60
+yarn.TimelineDataManagerMetrics.GetEntitiesTimeAvgTime
+yarn.TimelineDataManagerMetrics.GetEntitiesTotal
+yarn.TimelineDataManagerMetrics.PostEntitiesTimeAvgTime
+yarn.TimelineDataManagerMetrics.PostEntitiesTotal

ambari-metrics/ambari-metrics-timelineservice/conf/windows/metrics_whitelist

@@ -0,0 +1,527 @@
+AMBARI_METRICS.SmokeTest.FakeMetric
+ActiveThreads
+BytesQueued
+BytesReceivedLast5Minutes
+BytesSentLast5Minutes
+ChannelSize
+EventPutSuccessCount
+EventTakeSuccessCount
+FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Capacity
+FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.DfsUsed
+FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.NumFailedVolumes
+FlowFilesQueued
+FlowFilesReceivedLast5Minutes
+FlowFilesSentLast5Minutes
+Free Slots
+Supervisors
+TimelineMetricStoreWatcher.FakeMetric
+Topologies
+Total Executors
+Total Slots
+Total Tasks
+Used Slots
+bytes_in
+bytes_out
+cache.CacheMetrics.CacheCapacityRemaining
+cache.CacheMetrics.CacheCapacityRemainingPercentage
+cache.CacheMetrics.CacheCapacityTotal
+cache.CacheMetrics.CacheCapacityUsed
+cache.CacheMetrics.CacheHitRatio
+cache.CacheMetrics.CacheReadRequests
+cpu_idle
+cpu_intr
+cpu_nice
+cpu_num
+cpu_sintr
+cpu_steal
+cpu_system
+cpu_user
+cpu_wio
+default.General.active_calls_api_hs2_operation_INITIALIZED
+default.General.active_calls_api_hs2_operation_PENDING
+default.General.active_calls_api_hs2_operation_RUNNING
+default.General.api_create_table_count
+default.General.api_get_all_databases_75thpercentile
+default.General.api_get_all_databases_999thpercentile
+default.General.api_get_partitions_by_names_75thpercentile
+default.General.api_get_partitions_by_names_999thpercentile
+default.General.api_hs2_operation_INITIALIZED_count
+default.General.create_total_count_partitions
+default.General.hs2_completed_operation_CLOSED
+default.General.hs2_completed_operation_ERROR
+default.General.hs2_completed_operation_FINISHED
+default.General.init_total_count_dbs
+default.General.init_total_count_partitions
+default.General.init_total_count_tables
+default.General.jvm.pause.extraSleepTime
+default.General.memory.heap.committed
+default.General.memory.heap.max
+default.General.memory.heap.used
+default.General.memory.non-heap.committed
+default.General.memory.non-heap.max
+default.General.memory.non-heap.used
+dfs.FSNamesystem.CapacityTotalGB
+dfs.FSNamesystem.CapacityUsedGB
+dfs.FSNamesystem.ExpiredHeartbeats
+dfs.FSNamesystem.LastCheckpointTime
+dfs.FSNamesystem.LastWrittenTransactionId
+dfs.FSNamesystem.LockQueueLength
+dfs.FSNamesystem.MillisSinceLastLoadedEdits
+dfs.FSNamesystem.MissingReplOneBlocks
+dfs.FSNamesystem.NumFilesUnderConstruction
+dfs.FSNamesystem.PendingDataNodeMessageCount
+dfs.FSNamesystem.PostponedMisreplicatedBlocks
+dfs.FSNamesystem.Snapshots
+dfs.FSNamesystem.SnapshottableDirectories
+dfs.FSNamesystem.StaleDataNodes
+dfs.FSNamesystem.TotalFiles
+dfs.FSNamesystem.TotalLoad
+dfs.FSNamesystem.TransactionsSinceLastCheckpoint
+dfs.FSNamesystem.TransactionsSinceLastLogRoll
+dfs.FSNamesystem.UnderReplicatedBlocks
+dfs.datanode.BlocksRead
+dfs.datanode.BlocksWritten
+dfs.datanode.DatanodeNetworkErrors
+dfs.datanode.FsyncNanosAvgTime
+dfs.datanode.FsyncNanosNumOps
+dfs.datanode.PacketAckRoundTripTimeNanosAvgTime
+dfs.datanode.PacketAckRoundTripTimeNanosNumOps
+dfs.datanode.SendDataPacketBlockedOnNetworkNanosAvgTime
+dfs.datanode.SendDataPacketBlockedOnNetworkNanosNumOps
+dfs.datanode.SendDataPacketTransferNanosAvgTime
+dfs.datanode.SendDataPacketTransferNanosNumOps
+dfs.namenode.TotalFileOps
+disk_free
+disk_total
+disk_used
+executors.ExecutorMetrics.ExecutorAvailableFreeSlots
+executors.ExecutorMetrics.ExecutorAvailableFreeSlotsPercent
+executors.ExecutorMetrics.ExecutorCacheMemoryPerInstance
+executors.ExecutorMetrics.ExecutorJvmMaxMemory
+executors.ExecutorMetrics.ExecutorMaxFreeSlots
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost_300s50thPercentileLatency
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost_300s90thPercentileLatency
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost_300s95thPercentileLatency
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost_300s99thPercentileLatency
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill_300s50thPercentileLatency
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill_300s90thPercentileLatency
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill_300s95thPercentileLatency
+executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill_300s99thPercentileLatency
+executors.ExecutorMetrics.ExecutorMemoryPerInstance
+executors.ExecutorMetrics.ExecutorNumExecutorsPerInstance
+executors.ExecutorMetrics.ExecutorNumQueuedRequests
+executors.ExecutorMetrics.ExecutorTotalEvictedFromWaitQueue
+executors.ExecutorMetrics.ExecutorTotalFailed
+executors.ExecutorMetrics.ExecutorTotalKilled
+executors.ExecutorMetrics.ExecutorTotalRejectedRequests
+executors.ExecutorMetrics.ExecutorTotalRequestsHandled
+executors.ExecutorMetrics.ExecutorTotalSuccess
+gc.ConcurrentMarkSweep.count
+gc.ConcurrentMarkSweep.time
+gc.ParNew.count
+gc.ParNew.time
+io.IOMetrics.MaxDecodingTime
+io.IOMetrics.PercentileDecodingTime_30s50thPercentileLatency
+io.IOMetrics.PercentileDecodingTime_30s90thPercentileLatency
+io.IOMetrics.PercentileDecodingTime_30s95thPercentileLatency
+io.IOMetrics.PercentileDecodingTime_30s99thPercentileLatency
+ipc.client.org.apache.hadoop.ipc.DecayRpcScheduler.Caller(*).Priority
+ipc.client.org.apache.hadoop.ipc.DecayRpcScheduler.Caller(*).Volume
+jvm.JvmMetrics.GcCount
+jvm.JvmMetrics.GcCountConcurrentMarkSweep
+jvm.JvmMetrics.GcCountParNew
+jvm.JvmMetrics.GcNumWarnThresholdExceeded
+jvm.JvmMetrics.GcTimeMillis
+jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep
+jvm.JvmMetrics.GcTimeMillisParNew
+jvm.JvmMetrics.GcTotalExtraSleepTime
+jvm.JvmMetrics.LogError
+jvm.JvmMetrics.LogFatal
+jvm.JvmMetrics.MemHeapCommittedM
+jvm.JvmMetrics.MemHeapMaxM
+jvm.JvmMetrics.MemHeapUsedM
+jvm.JvmMetrics.MemNonHeapCommittedM
+jvm.JvmMetrics.MemNonHeapMaxM
+jvm.JvmMetrics.MemNonHeapUsedM
+jvm.JvmMetrics.ThreadsBlocked
+jvm.JvmMetrics.ThreadsNew
+jvm.JvmMetrics.ThreadsRunnable
+jvm.JvmMetrics.ThreadsTerminated
+jvm.JvmMetrics.ThreadsTimedWaiting
+jvm.JvmMetrics.ThreadsWaiting
+jvm.Master.JvmMetrics.ThreadsBlocked
+jvm.Master.JvmMetrics.ThreadsNew
+jvm.Master.JvmMetrics.ThreadsRunnable
+jvm.Master.JvmMetrics.ThreadsTerminated
+jvm.Master.JvmMetrics.ThreadsTimedWaiting
+jvm.Master.JvmMetrics.ThreadsWaiting
+jvm.RegionServer.JvmMetrics.GcCount
+jvm.RegionServer.JvmMetrics.GcCountConcurrentMarkSweep
+jvm.RegionServer.JvmMetrics.GcCountParNew
+jvm.RegionServer.JvmMetrics.GcTimeMillis
+jvm.RegionServer.JvmMetrics.GcTimeMillisConcurrentMarkSweep
+jvm.RegionServer.JvmMetrics.GcTimeMillisParNew
+jvm.RegionServer.JvmMetrics.MemHeapCommittedM
+jvm.RegionServer.JvmMetrics.MemHeapMaxM
+jvm.RegionServer.JvmMetrics.MemHeapUsedM
+jvm.RegionServer.JvmMetrics.MemNonHeapCommittedM
+jvm.RegionServer.JvmMetrics.MemNonHeapMaxM
+jvm.RegionServer.JvmMetrics.MemNonHeapUsedM
+jvm.RegionServer.JvmMetrics.ThreadsBlocked
+jvm.RegionServer.JvmMetrics.ThreadsNew
+jvm.RegionServer.JvmMetrics.ThreadsRunnable
+jvm.RegionServer.JvmMetrics.ThreadsTerminated
+jvm.RegionServer.JvmMetrics.ThreadsTimedWaiting
+jvm.RegionServer.JvmMetrics.ThreadsWaiting
+jvm.daemon_thread_count
+jvm.file_descriptor_usage
+jvm.heap_usage
+jvm.thread_count
+jvm.uptime
+kafka.controller.ControllerStats.LeaderElectionRateAndTimeMs.1MinuteRate
+kafka.controller.ControllerStats.LeaderElectionRateAndTimeMs.count
+kafka.controller.ControllerStats.UncleanLeaderElectionsPerSec.1MinuteRate
+kafka.controller.ControllerStats.UncleanLeaderElectionsPerSec.count
+kafka.controller.KafkaController.ActiveControllerCount
+kafka.controller.KafkaController.OfflinePartitionsCount
+kafka.log.LogFlushStats.LogFlushRateAndTimeMs.1MinuteRate
+kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.count
+kafka.network.RequestMetrics.RequestsPerSec.request.Produce.count
+kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.99percentile
+kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.max
+kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.mean
+kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.min
+kafka.network.RequestMetrics.TotalTimeMs.request.FetchFollower.99percentile
+kafka.network.RequestMetrics.TotalTimeMs.request.FetchFollower.max
+kafka.network.RequestMetrics.TotalTimeMs.request.FetchFollower.mean
+kafka.network.RequestMetrics.TotalTimeMs.request.FetchFollower.min
+kafka.network.RequestMetrics.TotalTimeMs.request.Produce.99percentile
+kafka.network.RequestMetrics.TotalTimeMs.request.Produce.max
+kafka.network.RequestMetrics.TotalTimeMs.request.Produce.mean
+kafka.network.RequestMetrics.TotalTimeMs.request.Produce.min
+kafka.server.BrokerTopicMetrics.BytesInPerSec.1MinuteRate
+kafka.server.BrokerTopicMetrics.BytesInPerSec.count
+kafka.server.BrokerTopicMetrics.BytesInPerSec.topic.*.count
+kafka.server.BrokerTopicMetrics.BytesOutPerSec.1MinuteRate
+kafka.server.BrokerTopicMetrics.BytesOutPerSec.count
+kafka.server.BrokerTopicMetrics.BytesOutPerSec.topic.*.count
+kafka.server.BrokerTopicMetrics.MessagesInPerSec.1MinuteRate
+kafka.server.BrokerTopicMetrics.MessagesInPerSec.count
+kafka.server.BrokerTopicMetrics.MessagesInPerSec.topic.*.count
+kafka.server.BrokerTopicMetrics.TotalProduceRequestsPerSec.topic.*.count
+kafka.server.FetcherLagMetrics.ConsumerLag.clientId.ReplicaFetcherThread-0-1001.partition.0.topic.*
+kafka.server.ReplicaFetcherManager.MaxLag.clientId.Replica
+kafka.server.ReplicaManager.IsrExpandsPerSec.count
+kafka.server.ReplicaManager.IsrShrinksPerSec.count
+kafka.server.ReplicaManager.LeaderCount
+kafka.server.ReplicaManager.PartitionCount
+kafka.server.ReplicaManager.UnderReplicatedPartitions
+load_fifteen
+load_five
+load_one
+master.AssignmentManger.ritCount
+master.AssignmentManger.ritCountOverThreshold
+master.AssignmentManger.ritOldestAge
+master.Master.QueueCallTime_median
+master.Master.TotalCallTime_num_ops
+master.Master.exceptions
+master.Server.averageLoad
+master.Server.numDeadRegionServers
+master.Server.numRegionServers
+mem_buffered
+mem_cached
+mem_free
+mem_shared
+mem_total
+mem_used
+memory.heap.committed
+memory.heap.max
+memory.heap.used
+memory.non-heap.committed
+memory.non-heap.max
+memory.non-heap.used
+pkts_in
+pkts_out
+proc_run
+proc_total
+read_bps
+read_bytes
+read_count
+read_time
+regionserver.RegionServer.ProcessCallTime_75th_percentile
+regionserver.RegionServer.ProcessCallTime_95th_percentile
+regionserver.RegionServer.ProcessCallTime_99th_percentile
+regionserver.RegionServer.ProcessCallTime_max
+regionserver.RegionServer.ProcessCallTime_mean
+regionserver.RegionServer.ProcessCallTime_median
+regionserver.RegionServer.QueueCallTime_75th_percentile
+regionserver.RegionServer.QueueCallTime_95th_percentile
+regionserver.RegionServer.QueueCallTime_99th_percentile
+regionserver.RegionServer.QueueCallTime_max
+regionserver.RegionServer.QueueCallTime_mean
+regionserver.RegionServer.QueueCallTime_median
+regionserver.RegionServer.TotalCallTime_num_ops
+regionserver.RegionServer.authenticationFailures
+regionserver.RegionServer.authenticationSuccesses
+regionserver.RegionServer.authorizationFailures
+regionserver.RegionServer.authorizationSuccesses
+regionserver.RegionServer.exceptions
+regionserver.RegionServer.numActiveHandler
+regionserver.RegionServer.numCallsInGeneralQueue
+regionserver.RegionServer.numCallsInPriorityQueue
+regionserver.RegionServer.numCallsInReplicationQueue
+regionserver.RegionServer.numOpenConnections
+regionserver.RegionServer.queueSize
+regionserver.RegionServer.receivedBytes
+regionserver.RegionServer.sentBytes
+regionserver.Server.Append_75th_percentile
+regionserver.Server.Append_95th_percentile
+regionserver.Server.Append_99th_percentile
+regionserver.Server.Append_max
+regionserver.Server.Append_mean
+regionserver.Server.Append_median
+regionserver.Server.Append_min
+regionserver.Server.Append_num_ops
+regionserver.Server.Delete_75th_percentile
+regionserver.Server.Delete_95th_percentile
+regionserver.Server.Delete_99th_percentile
+regionserver.Server.Delete_max
+regionserver.Server.Delete_mean
+regionserver.Server.Delete_median
+regionserver.Server.Delete_min
+regionserver.Server.Delete_num_ops
+regionserver.Server.Get_75th_percentile
+regionserver.Server.Get_95th_percentile
+regionserver.Server.Get_99th_percentile
+regionserver.Server.Get_max
+regionserver.Server.Get_mean
+regionserver.Server.Get_median
+regionserver.Server.Get_min
+regionserver.Server.Get_num_ops
+regionserver.Server.Increment_75th_percentile
+regionserver.Server.Increment_95th_percentile
+regionserver.Server.Increment_99th_percentile
+regionserver.Server.Increment_max
+regionserver.Server.Increment_mean
+regionserver.Server.Increment_median
+regionserver.Server.Increment_min
+regionserver.Server.Increment_num_ops
+regionserver.Server.Mutate_75th_percentile
+regionserver.Server.Mutate_95th_percentile
+regionserver.Server.Mutate_99th_percentile
+regionserver.Server.Mutate_max
+regionserver.Server.Mutate_mean
+regionserver.Server.Mutate_median
+regionserver.Server.Mutate_min
+regionserver.Server.Mutate_num_ops
+regionserver.Server.Replay_75th_percentile
+regionserver.Server.Replay_95th_percentile
+regionserver.Server.Replay_99th_percentile
+regionserver.Server.Replay_max
+regionserver.Server.Replay_mean
+regionserver.Server.Replay_median
+regionserver.Server.Replay_min
+regionserver.Server.Replay_num_ops
+regionserver.Server.ScanNext_95th_percentile
+regionserver.Server.ScanNext_num_ops
+regionserver.Server.ScanTime_75th_percentile
+regionserver.Server.ScanTime_95th_percentile
+regionserver.Server.ScanTime_99th_percentile
+regionserver.Server.ScanTime_max
+regionserver.Server.ScanTime_mean
+regionserver.Server.ScanTime_median
+regionserver.Server.blockCacheCount
+regionserver.Server.blockCacheCountHitPercent
+regionserver.Server.blockCacheEvictionCount
+regionserver.Server.blockCacheExpressHitPercent
+regionserver.Server.blockCacheFreeSize
+regionserver.Server.blockCacheHitCount
+regionserver.Server.blockCacheMissCount
+regionserver.Server.blockCacheSize
+regionserver.Server.blockCountHitPercent
+regionserver.Server.compactionQueueLength
+regionserver.Server.flushQueueLength
+regionserver.Server.hlogFileCount
+regionserver.Server.hlogFileSize
+regionserver.Server.memStoreSize
+regionserver.Server.percentFilesLocal
+regionserver.Server.readRequestCount
+regionserver.Server.regionCount
+regionserver.Server.slowDeleteCount
+regionserver.Server.slowGetCount
+regionserver.Server.slowPutCount
+regionserver.Server.splitQueueLength
+regionserver.Server.staticBloomSize
+regionserver.Server.staticIndexSize
+regionserver.Server.storeCount
+regionserver.Server.storeFileCount
+regionserver.Server.storeFileIndexSize
+regionserver.Server.storeFileSize
+regionserver.Server.totalRequestCount
+regionserver.Server.updatesBlockedTime
+regionserver.Server.writeRequestCount
+regionserver.Tables.*_metric_averageRegionSize
+regionserver.Tables.*_metric_avgStoreFileAge
+regionserver.Tables.*_metric_flushMemstoreSize_95th_percentile
+regionserver.Tables.*_metric_flushMemstoreSize_max
+regionserver.Tables.*_metric_flushMemstoreSize_mean
+regionserver.Tables.*_metric_flushMemstoreSize_median
+regionserver.Tables.*_metric_flushOutputSize_95th_percentile
+regionserver.Tables.*_metric_flushOutputSize_max
+regionserver.Tables.*_metric_flushOutputSize_mean
+regionserver.Tables.*_metric_flushOutputSize_median
+regionserver.Tables.*_metric_flushTime_95th_percentile
+regionserver.Tables.*_metric_flushTime_max
+regionserver.Tables.*_metric_flushTime_mean
+regionserver.Tables.*_metric_flushTime_median
+regionserver.Tables.*_metric_flushTime_num_ops
+regionserver.Tables.*_metric_flushedMemstoreBytes
+regionserver.Tables.*_metric_flushedOutputBytes
+regionserver.Tables.*_metric_maxStoreFileAge
+regionserver.Tables.*_metric_memStoreSize
+regionserver.Tables.*_metric_minStoreFileAge
+regionserver.Tables.*_metric_numReferenceFiles
+regionserver.Tables.*_metric_readRequestCount
+regionserver.Tables.*_metric_regionCount
+regionserver.Tables.*_metric_storeCount
+regionserver.Tables.*_metric_storeFileCount
+regionserver.Tables.*_metric_storeFileSize
+regionserver.Tables.*_metric_tableSize
+regionserver.Tables.*_metric_totalRequestCount
+regionserver.Tables.*_metric_writeRequestCount
+regionserver.Users.*_metric_append_num_ops
+regionserver.Users.*_metric_delete_num_ops
+regionserver.Users.*_metric_get_num_ops
+regionserver.Users.*_metric_increment_num_ops
+regionserver.Users.*_metric_mutate_num_ops
+regionserver.Users.*_metric_scanTime_num_ops
+regionserver.WAL.AppendSize_75th_percentile
+regionserver.WAL.AppendSize_95th_percentile
+regionserver.WAL.AppendSize_99th_percentile
+regionserver.WAL.AppendSize_max
+regionserver.WAL.AppendSize_mean
+regionserver.WAL.AppendSize_median
+regionserver.WAL.AppendTime_75th_percentile
+regionserver.WAL.AppendTime_95th_percentile
+regionserver.WAL.AppendTime_99th_percentile
+regionserver.WAL.AppendTime_max
+regionserver.WAL.AppendTime_mean
+regionserver.WAL.AppendTime_median
+regionserver.WAL.SyncTime_75th_percentile
+regionserver.WAL.SyncTime_95th_percentile
+regionserver.WAL.SyncTime_99th_percentile
+regionserver.WAL.SyncTime_max
+regionserver.WAL.SyncTime_mean
+regionserver.WAL.SyncTime_median
+regionserver.WAL.SyncTime_num_ops
+regionserver.WAL.appendCount
+regionserver.WAL.slowAppendCount
+rpc.rpc.CallQueueLength
+rpc.rpc.NumOpenConnections
+rpc.rpc.RpcAuthenticationFailures
+rpc.rpc.RpcAuthorizationFailures
+rpc.rpc.RpcProcessingTimeAvgTime
+rpc.rpc.RpcQueueTimeAvgTime
+rpc.rpc.RpcSlowCalls
+rpc.rpc.client.CallQueueLength
+rpc.rpc.client.NumOpenConnections
+rpc.rpc.client.RpcClientBackoff
+rpc.rpc.client.RpcProcessingTimeAvgTime
+rpc.rpc.client.RpcProcessingTimeNumOps
+rpc.rpc.client.RpcQueueTimeAvgTime
+rpc.rpc.client.RpcQueueTimeNumOps
+rpc.rpc.client.RpcSlowCalls
+rpc.rpc.datanode.CallQueueLength
+rpc.rpc.datanode.RpcProcessingTimeAvgTime
+rpc.rpc.datanode.RpcProcessingTimeNumOps
+rpc.rpc.datanode.RpcQueueTimeAvgTime
+rpc.rpc.datanode.RpcQueueTimeNumOps
+rpc.rpc.datanode.RpcSlowCalls
+rpcdetailed.rpcdetailed.client.AddBlockAvgTime
+rpcdetailed.rpcdetailed.client.AddBlockNumOps
+swap_free
+swap_total
+threads.blocked.count
+threads.count
+threads.daemon.count
+threads.deadlock.count
+threads.runnable.count
+topology.*.%.--ack-count.%
+topology.*.%.--complete-latency.%
+topology.*.%.--emit-count.%
+topology.*.%.--execute-count.%
+topology.*.%.--fail-count.%
+topology.*.%.--receive.population
+topology.*.%.--sendqueue.population
+topology.*.*.%.--ack-count.%
+topology.*.*.%.--complete-latency.%
+topology.*.*.%.--emit-count.%
+topology.*.*.%.--execute-count.%
+topology.*.*.%.--fail-count.%
+topology.*.*.%.--process-latency.%
+topology.*.*.%.--receive.population
+topology.*.*.%.--sendqueue.population
+topology.*.--system.%.%.-1.GC/PSMarkSweep.count
+topology.*.--system.%.%.-1.GC/PSMarkSweep.timeMs
+topology.*.--system.%.%.-1.GC/PSScavenge.count
+topology.*.--system.%.%.-1.GC/PSScavenge.timeMs
+topology.*.--system.%.%.-1.memory/heap.usedBytes
+topology.*.--system.%.%.-1.memory/nonHeap.usedBytes
+topology.*.kafka-topic.*.*.latestCompletedOffset
+topology.*.kafka-topic.*.*.latestTimeOffset
+write_bps
+write_bytes
+write_count
+write_time
+yarn.ClusterMetrics.AMLaunchDelayAvgTime
+yarn.ClusterMetrics.AMRegisterDelayAvgTime
+yarn.ClusterMetrics.NumActiveNMs
+yarn.ClusterMetrics.NumLostNMs
+yarn.ClusterMetrics.NumUnhealthyNMs
+yarn.NodeManagerMetrics.AllocatedContainers
+yarn.NodeManagerMetrics.AllocatedGB
+yarn.NodeManagerMetrics.AllocatedVCores
+yarn.NodeManagerMetrics.AvailableGB
+yarn.NodeManagerMetrics.BadLocalDirs
+yarn.NodeManagerMetrics.BadLogDirs
+yarn.NodeManagerMetrics.ContainerLaunchDurationAvgTime
+yarn.NodeManagerMetrics.ContainersCompleted
+yarn.NodeManagerMetrics.ContainersFailed
+yarn.NodeManagerMetrics.ContainersIniting
+yarn.NodeManagerMetrics.ContainersKilled
+yarn.NodeManagerMetrics.ContainersLaunched
+yarn.NodeManagerMetrics.ContainersRunning
+yarn.NodeManagerMetrics.GoodLocalDirsDiskUtilizationPerc
+yarn.NodeManagerMetrics.GoodLogDirsDiskUtilizationPerc
+yarn.QueueMetrics.Queue=root.AMResourceLimitMB
+yarn.QueueMetrics.Queue=root.AggregateContainersAllocated
+yarn.QueueMetrics.Queue=root.AggregateNodeLocalContainersAllocated
+yarn.QueueMetrics.Queue=root.AggregateOffSwitchContainersAllocated
+yarn.QueueMetrics.Queue=root.AggregateRackLocalContainersAllocated
+yarn.QueueMetrics.Queue=root.AllocatedContainers
+yarn.QueueMetrics.Queue=root.AllocatedMB
+yarn.QueueMetrics.Queue=root.AllocatedVCores
+yarn.QueueMetrics.Queue=root.AppAttemptFirstContainerAllocationDelayAvgTime
+yarn.QueueMetrics.Queue=root.AppsCompleted
+yarn.QueueMetrics.Queue=root.AppsFailed
+yarn.QueueMetrics.Queue=root.AppsKilled
+yarn.QueueMetrics.Queue=root.AppsPending
+yarn.QueueMetrics.Queue=root.AppsRunning
+yarn.QueueMetrics.Queue=root.AppsSubmitted
+yarn.QueueMetrics.Queue=root.AvailableMB
+yarn.QueueMetrics.Queue=root.AvailableVCores
+yarn.QueueMetrics.Queue=root.PendingContainers
+yarn.QueueMetrics.Queue=root.PendingMB
+yarn.QueueMetrics.Queue=root.UsedAMResourceMB
+yarn.QueueMetrics.Queue=root.default.ReservedContainers
+yarn.QueueMetrics.Queue=root.default.ReservedMB
+yarn.QueueMetrics.Queue=root.running_0
+yarn.QueueMetrics.Queue=root.running_1440
+yarn.QueueMetrics.Queue=root.running_300
+yarn.QueueMetrics.Queue=root.running_60
+yarn.TimelineDataManagerMetrics.GetEntitiesTimeAvgTime
+yarn.TimelineDataManagerMetrics.GetEntitiesTotal
+yarn.TimelineDataManagerMetrics.PostEntitiesTimeAvgTime
+yarn.TimelineDataManagerMetrics.PostEntitiesTotal

ambari-metrics/ambari-metrics-timelineservice/pom.xml

@@ -207,6 +207,9 @@
                 <source>
                   <location>conf/unix/log4j.properties</location>
                 </source>
+                <source>
+                  <location>conf/unix/metrics_whitelist</location>
+                </source>
                 <source>
                   <location>target/embedded/${hbase.folder}/conf/hbase-site.xml</location>
                 </source>
@@ -243,6 +246,24 @@
           <groups>${testcase.groups}</groups>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+        <configuration>
+          <excludes>
+            <exclude>conf/unix/metrics_whitelist</exclude>
+            <exclude>conf/windows/metrics_whitelist</exclude>
+          </excludes>
+        </configuration>
+        <executions>
+          <execution>
+            <phase>test</phase>
+            <goals>
+              <goal>check</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>org.vafer</groupId>
         <artifactId>jdeb</artifactId>

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java

@@ -31,7 +31,6 @@ import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
 import org.apache.hadoop.metrics2.sink.timeline.TopNConfig;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineMetricAggregator;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineMetricAggregatorFactory;
@@ -64,7 +63,6 @@ import java.util.concurrent.TimeUnit;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.AGGREGATOR_CHECKPOINT_DELAY;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.DEFAULT_TOPN_HOSTS_LIMIT;
-import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_WHITELIST_FILE;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.USE_GROUPBY_AGGREGATOR_QUERIES;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.ACTUAL_AGGREGATOR_NAMES;
 
@@ -117,10 +115,8 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin
           "initialize HA controller", e);
       }
 
-      String whitelistFile = metricsConf.get(TIMELINE_METRICS_WHITELIST_FILE, "");
-      if (!StringUtils.isEmpty(whitelistFile)) {
-        AggregatorUtils.populateMetricWhitelistFromFile(whitelistFile);
-      }
+      //Initialize whitelisting & blacklisting if needed
+      TimelineMetricsFilter.initializeMetricFilter(metricsConf);
 
       defaultTopNHostsLimit = Integer.parseInt(metricsConf.get(DEFAULT_TOPN_HOSTS_LIMIT, "20"));
       if (Boolean.parseBoolean(metricsConf.get(USE_GROUPBY_AGGREGATOR_QUERIES, "true"))) {
@@ -359,13 +355,19 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin
   }
 
   @Override
-  public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata() throws SQLException, IOException {
+  public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata(String query) throws SQLException, IOException {
     Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata =
       metricMetadataManager.getMetadataCache();
 
+    boolean includeBlacklistedMetrics = StringUtils.isNotEmpty(query) && "all".equalsIgnoreCase(query);
+
     // Group Metadata by AppId
     Map<String, List<TimelineMetricMetadata>> metadataByAppId = new HashMap<>();
     for (TimelineMetricMetadata metricMetadata : metadata.values()) {
+
+      if (!includeBlacklistedMetrics && !metricMetadata.isWhitelisted()) {
+        continue;
+      }
       List<TimelineMetricMetadata> metadataList = metadataByAppId.get(metricMetadata.getAppId());
       if (metadataList == null) {
         metadataList = new ArrayList<>();

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java

@@ -18,7 +18,6 @@
 package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
 
 import com.google.common.collect.Multimap;
-import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -111,6 +110,7 @@ import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.ti
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_METRICS_CLUSTER_AGGREGATE_GROUPED_TABLE_SQL;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_METRICS_CLUSTER_AGGREGATE_TABLE_SQL;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_METRICS_METADATA_TABLE_SQL;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.ALTER_METRICS_METADATA_TABLE;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_METRICS_TABLE_SQL;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.DEFAULT_ENCODING;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.DEFAULT_TABLE_COMPRESSION;
@@ -403,6 +403,8 @@ public class PhoenixHBaseAccessor {
       String metadataSql = String.format(CREATE_METRICS_METADATA_TABLE_SQL,
         encoding, compression);
       stmt.executeUpdate(metadataSql);
+      stmt.executeUpdate(ALTER_METRICS_METADATA_TABLE);
+
       String hostedAppSql = String.format(CREATE_HOSTED_APPS_METADATA_TABLE_SQL,
         encoding, compression);
       stmt.executeUpdate(hostedAppSql);
@@ -718,20 +720,19 @@ public class PhoenixHBaseAccessor {
 
       TimelineMetric tm = iterator.next();
 
-      if (CollectionUtils.isNotEmpty(AggregatorUtils.whitelistedMetrics) &&
-        !AggregatorUtils.whitelistedMetrics.contains(tm.getMetricName())) {
-        iterator.remove();
-        continue;
-      }
+      boolean acceptMetric = TimelineMetricsFilter.acceptMetric(tm);
 
       // Write to metadata cache on successful write to store
       if (metadataManager != null) {
         metadataManager.putIfModifiedTimelineMetricMetadata(
-                metadataManager.getTimelineMetricMetadata(tm));
+                metadataManager.getTimelineMetricMetadata(tm, acceptMetric));
 
         metadataManager.putIfModifiedHostedAppsMetadata(
                 tm.getHostName(), tm.getAppId());
       }
+      if (!acceptMetric) {
+        iterator.remove();
+      }
     }
 
     if  (!skipCache && cacheEnabled) {
@@ -1469,6 +1470,7 @@ public class PhoenixHBaseAccessor {
         stmt.setString(4, metadata.getType());
         stmt.setLong(5, metadata.getSeriesStartTime());
         stmt.setBoolean(6, metadata.isSupportsAggregates());
+        stmt.setBoolean(7, metadata.isWhitelisted());
 
         try {
           stmt.executeUpdate();
@@ -1561,7 +1563,8 @@ public class PhoenixHBaseAccessor {
           rs.getString("UNITS"),
           rs.getString("TYPE"),
           rs.getLong("START_TIME"),
-          rs.getBoolean("SUPPORTS_AGGREGATION")
+          rs.getBoolean("SUPPORTS_AGGREGATION"),
+          rs.getBoolean("IS_WHITELISTED")
         );
 
         TimelineMetricMetadataKey key = new TimelineMetricMetadataKey(metricName, appId);

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java

@@ -243,6 +243,9 @@ public class TimelineMetricConfiguration {
   public static final String TIMELINE_METRICS_WHITELIST_FILE =
     "timeline.metrics.whitelist.file";
 
+  public static final String TIMELINE_METRICS_APPS_BLACKLIST =
+    "timeline.metrics.apps.blacklist";
+
   public static final String HBASE_BLOCKING_STORE_FILES =
     "hbase.hstore.blockingStoreFiles";
 

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java

@@ -78,7 +78,7 @@ public interface TimelineMetricStore {
    * @throws SQLException
    * @throws IOException
    */
-  Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata() throws SQLException, IOException;
+  Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata(String query) throws SQLException, IOException;
 
   /**
    * Returns all hosts that have written metrics with the apps on the host

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java

@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_WHITELIST_FILE;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_APPS_BLACKLIST;
+
+public class TimelineMetricsFilter {
+
+  private static Set<String> whitelistedMetrics;
+  private static Set<Pattern> whitelistedMetricPatterns;
+  private static boolean metricWhitelistingEnabled;
+  private static boolean appBlacklistingEnabled;
+  private static Set<String> blacklistedApps;
+  private static String patternPrefix = "._p_";
+
+  private static final Log LOG = LogFactory.getLog(TimelineMetricsFilter.class);
+
+  public static void initializeMetricFilter(Configuration metricsConf) {
+
+    whitelistedMetrics = new HashSet<String>();
+    whitelistedMetricPatterns = new HashSet<Pattern>();
+    blacklistedApps = new HashSet<>();
+    metricWhitelistingEnabled = false;
+    appBlacklistingEnabled = false;
+
+    String whitelistFile = metricsConf.get(TIMELINE_METRICS_WHITELIST_FILE, "");
+    if (!StringUtils.isEmpty(whitelistFile)) {
+      metricWhitelistingEnabled = true;
+      readMetricWhitelistFromFile(whitelistFile);
+    }
+
+    String appsBlacklist = metricsConf.get(TIMELINE_METRICS_APPS_BLACKLIST, "");
+    if (!StringUtils.isEmpty(appsBlacklist)) {
+      appBlacklistingEnabled = true;
+      for (String app : appsBlacklist.split(",")) {
+        blacklistedApps.add(app);
+      }
+      LOG.info("Blacklisted apps : " + blacklistedApps.toString());
+    }
+  }
+
+  private static void readMetricWhitelistFromFile(String whitelistFile) {
+
+    BufferedReader br = null;
+    String strLine;
+
+    try(FileInputStream fstream = new FileInputStream(whitelistFile)) {
+      br = new BufferedReader(new InputStreamReader(fstream));
+
+      while ((strLine = br.readLine()) != null)   {
+        strLine = strLine.trim();
+        if (StringUtils.isEmpty(strLine)) {
+          continue;
+        }
+        if (strLine.startsWith(patternPrefix)) {
+          whitelistedMetricPatterns.add(Pattern.compile(strLine.substring(patternPrefix.length())));
+        } else {
+          whitelistedMetrics.add(strLine);
+        }
+      }
+    } catch (IOException ioEx) {
+      LOG.error("Unable to parse metric whitelist file", ioEx);
+    }
+
+    LOG.info("Whitelisting " + whitelistedMetrics.size() + " metrics");
+    LOG.debug("Whitelisted metrics : " + Arrays.toString(whitelistedMetrics.toArray()));
+  }
+
+  public static boolean acceptMetric(String metricName, String appId) {
+    TimelineMetric timelineMetric = new TimelineMetric();
+    timelineMetric.setAppId(appId);
+    timelineMetric.setMetricName(metricName);
+    return acceptMetric(timelineMetric);
+  }
+
+  public static boolean acceptMetric(TimelineMetric metric) {
+
+    if (!isEnabled()) {
+      return true;
+    }
+
+    // App takes precedence.
+    if (blacklistedApps.contains(metric.getAppId())) {
+      return false;
+    }
+
+    if (!metricWhitelistingEnabled) {
+      return true;
+    }
+
+    if (whitelistedMetrics.contains(metric.getMetricName())) {
+      return true;
+    }
+
+    for (Pattern p : whitelistedMetricPatterns) {
+      Matcher m = p.matcher(metric.getMetricName());
+      if (m.find()) {
+        whitelistedMetrics.add(metric.getMetricName());
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  public static void addToWhitelist(String metricName) {
+
+    if (StringUtils.isNotEmpty(metricName)) {
+      if (!metricWhitelistingEnabled)  {
+        metricWhitelistingEnabled = true;
+      }
+
+      whitelistedMetrics.add(metricName);
+    }
+  }
+
+  public static boolean isEnabled() {
+    return appBlacklistingEnabled || metricWhitelistingEnabled;
+  }
+}

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AggregatorUtils.java

@@ -18,15 +18,7 @@
 package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators;
 
 
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.Arrays;
-import java.util.HashSet;
 import java.util.Map;
-import java.util.Set;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -35,7 +27,6 @@ import org.apache.commons.logging.LogFactory;
  */
 public class AggregatorUtils {
 
-  public static Set<String> whitelistedMetrics = new HashSet<String>();
   private static final Log LOG = LogFactory.getLog(AggregatorUtils.class);
 
   public static double[] calculateAggregates(Map<Long, Double> metricValues) {
@@ -68,40 +59,4 @@ public class AggregatorUtils {
 
     return values;
   }
-
-  public static void populateMetricWhitelistFromFile(String whitelistFile) {
-
-    FileInputStream fstream = null;
-    BufferedReader br = null;
-    String strLine;
-
-    try {
-      fstream = new FileInputStream(whitelistFile);
-      br = new BufferedReader(new InputStreamReader(fstream));
-
-      while ((strLine = br.readLine()) != null)   {
-        strLine = strLine.trim();
-        whitelistedMetrics.add(strLine);
-      }
-    } catch (IOException ioEx) {
-      LOG.error("Unable to parse metric whitelist file", ioEx);
-    } finally {
-      if (br != null) {
-        try {
-          br.close();
-        } catch (IOException e) {
-        }
-      }
-
-      if (fstream != null) {
-        try {
-          fstream.close();
-        } catch (IOException e) {
-        }
-      }
-    }
-    LOG.info("Whitelisting " + whitelistedMetrics.size() + " metrics");
-    LOG.debug("Whitelisted metrics : " + Arrays.toString(whitelistedMetrics.toArray()));
-  }
-
 }

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java

@@ -21,12 +21,11 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
 import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricsFilter;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataKey;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -141,7 +140,7 @@ public class TimelineMetricAppAggregator {
           if (hostMetricMetadata != null) {
             TimelineMetricMetadata timelineMetricMetadata = new TimelineMetricMetadata(clusterMetric.getMetricName(),
               appId, hostMetricMetadata.getUnits(), hostMetricMetadata.getType(), hostMetricMetadata.getSeriesStartTime(),
-              hostMetricMetadata.isSupportsAggregates());
+              hostMetricMetadata.isSupportsAggregates(), TimelineMetricsFilter.acceptMetric(clusterMetric.getMetricName(), appId));
             metadataManagerInstance.putIfModifiedTimelineMetricMetadata(timelineMetricMetadata);
           }
         }

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java

@@ -196,14 +196,15 @@ public class TimelineMetricMetadataManager {
     hBaseAccessor.saveHostAppsMetadata(hostedApps);
   }
 
-  public TimelineMetricMetadata getTimelineMetricMetadata(TimelineMetric timelineMetric) {
+  public TimelineMetricMetadata getTimelineMetricMetadata(TimelineMetric timelineMetric, boolean isWhitelisted) {
     return new TimelineMetricMetadata(
       timelineMetric.getMetricName(),
       timelineMetric.getAppId(),
       timelineMetric.getUnits(),
       timelineMetric.getType(),
       timelineMetric.getStartTime(),
-      supportAggregates(timelineMetric)
+      supportAggregates(timelineMetric),
+      isWhitelisted
     );
   }
 

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java

@@ -137,7 +137,8 @@ public class PhoenixTransactSQL {
       "UNITS CHAR(20), " +
       "TYPE CHAR(20), " +
       "START_TIME UNSIGNED_LONG, " +
-      "SUPPORTS_AGGREGATION BOOLEAN " +
+      "SUPPORTS_AGGREGATION BOOLEAN, " +
+      "IS_WHITELISTED BOOLEAN " +
       "CONSTRAINT pk PRIMARY KEY (METRIC_NAME, APP_ID)) " +
       "DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'";
 
@@ -147,6 +148,9 @@ public class PhoenixTransactSQL {
       "CONSTRAINT pk PRIMARY KEY (HOSTNAME))" +
       "DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'";
 
+  public static final String ALTER_METRICS_METADATA_TABLE =
+    "ALTER TABLE METRICS_METADATA ADD IF NOT EXISTS IS_WHITELISTED BOOLEAN";
+
   /**
    * ALTER table to set new options
    */
@@ -220,8 +224,8 @@ public class PhoenixTransactSQL {
 
   public static final String UPSERT_METADATA_SQL =
     "UPSERT INTO METRICS_METADATA (METRIC_NAME, APP_ID, UNITS, TYPE, " +
-      "START_TIME, SUPPORTS_AGGREGATION) " +
-      "VALUES (?, ?, ?, ?, ?, ?)";
+      "START_TIME, SUPPORTS_AGGREGATION, IS_WHITELISTED) " +
+      "VALUES (?, ?, ?, ?, ?, ?, ?)";
 
   public static final String UPSERT_HOSTED_APPS_METADATA_SQL =
     "UPSERT INTO HOSTED_APPS_METADATA (HOSTNAME, APP_IDS) VALUES (?, ?)";
@@ -300,7 +304,7 @@ public class PhoenixTransactSQL {
 
   public static final String GET_METRIC_METADATA_SQL = "SELECT " +
     "METRIC_NAME, APP_ID, UNITS, TYPE, START_TIME, " +
-    "SUPPORTS_AGGREGATION FROM METRICS_METADATA";
+    "SUPPORTS_AGGREGATION, IS_WHITELISTED FROM METRICS_METADATA";
 
   public static final String GET_HOSTED_APPS_METADATA_SQL = "SELECT " +
     "HOSTNAME, APP_IDS FROM HOSTED_APPS_METADATA";

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java

@@ -384,12 +384,13 @@ public class TimelineWebServices {
   @Produces({ MediaType.APPLICATION_JSON })
   public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata(
     @Context HttpServletRequest req,
-    @Context HttpServletResponse res
-  ) {
+    @Context HttpServletResponse res,
+    @QueryParam("query") String query
+    ) {
     init(res);
 
     try {
-      return timelineMetricStore.getTimelineMetricMetadata();
+      return timelineMetricStore.getTimelineMetricMetadata(query);
     } catch (Exception e) {
       throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
     }

ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java

@@ -87,7 +87,7 @@ public class TestTimelineMetricStore implements TimelineMetricStore {
   }
 
   @Override
-  public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata() throws SQLException, IOException {
+  public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata(String query) throws SQLException, IOException {
     return null;
   }
 

ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java

@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
+
+import junit.framework.Assert;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.junit.Test;
+import java.net.URL;
+
+public class TimelineMetricsFilterTest {
+
+  @Test
+  public void testAppBlacklisting() {
+
+    Configuration metricsConf = new Configuration();
+    metricsConf.set("timeline.metrics.apps.blacklist", "hbase,datanode,nimbus");
+    TimelineMetricsFilter.initializeMetricFilter(metricsConf);
+
+    TimelineMetric timelineMetric = new TimelineMetric();
+
+    timelineMetric.setAppId("hbase");
+    Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+    timelineMetric.setAppId("namenode");
+    Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+    timelineMetric.setAppId("nimbus");
+    Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
+  }
+
+  @Test
+  public void testMetricWhitelisting() throws Exception {
+
+    Configuration metricsConf = new Configuration();
+    URL fileUrl = ClassLoader.getSystemResource("test_data/metric_whitelist.dat");
+
+    metricsConf.set("timeline.metrics.whitelist.file", fileUrl.getPath());
+    TimelineMetricsFilter.initializeMetricFilter(metricsConf);
+
+    TimelineMetric timelineMetric = new TimelineMetric();
+
+    timelineMetric.setMetricName("cpu_system");
+    Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+    timelineMetric.setMetricName("cpu_system1");
+    Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+    timelineMetric.setMetricName("jvm.JvmMetrics.MemHeapUsedM");
+    Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+    timelineMetric.setMetricName("dfs.FSNamesystem.TotalFiles");
+    Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+  }
+
+  @Test
+  public void testTogether() throws Exception {
+
+    Configuration metricsConf = new Configuration();
+    metricsConf.set("timeline.metrics.apps.blacklist", "hbase,datanode,nimbus");
+
+    URL fileUrl = ClassLoader.getSystemResource("test_data/metric_whitelist.dat");
+    metricsConf.set("timeline.metrics.whitelist.file", fileUrl.getPath());
+
+    TimelineMetricsFilter.initializeMetricFilter(metricsConf);
+
+    TimelineMetric timelineMetric = new TimelineMetric();
+
+    timelineMetric.setMetricName("cpu_system");
+    timelineMetric.setAppId("hbase");
+    Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+    timelineMetric.setMetricName("cpu_system");
+    timelineMetric.setAppId("HOST");
+    Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+    timelineMetric.setMetricName("jvm.JvmMetrics.MemHeapUsedM");
+    Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+    timelineMetric.setMetricName("dfs.FSNamesystem.TotalFiles");
+    Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+  }
+
+}

ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java

@@ -23,14 +23,12 @@ import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
 import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
 import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.AbstractMiniHBaseClusterTest;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricsFilter;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.io.IOException;
 import java.sql.SQLException;
-import java.util.Iterator;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
@@ -88,8 +86,9 @@ public class TestMetadataManager extends AbstractMiniHBaseClusterTest {
     }});
     timelineMetrics.getMetrics().add(metric3);
 
-    AggregatorUtils.whitelistedMetrics.add("dummy_metric1");
-    AggregatorUtils.whitelistedMetrics.add("dummy_metric2");
+    TimelineMetricsFilter.initializeMetricFilter(new Configuration());
+    TimelineMetricsFilter.addToWhitelist("dummy_metric1");
+    TimelineMetricsFilter.addToWhitelist("dummy_metric2");
 
     hdb.insertMetricRecordsWithMetadata(metadataManager, timelineMetrics, true);
   }
@@ -99,16 +98,20 @@ public class TestMetadataManager extends AbstractMiniHBaseClusterTest {
     Map<TimelineMetricMetadataKey, TimelineMetricMetadata> cachedData = metadataManager.getMetadataCache();
 
     Assert.assertNotNull(cachedData);
-    Assert.assertEquals(2, cachedData.size());
+    Assert.assertEquals(3, cachedData.size());
     TimelineMetricMetadataKey key1 = new TimelineMetricMetadataKey("dummy_metric1", "dummy_app1");
     TimelineMetricMetadataKey key2 = new TimelineMetricMetadataKey("dummy_metric2", "dummy_app2");
+    TimelineMetricMetadataKey key3 = new TimelineMetricMetadataKey("dummy_metric3", "dummy_app3");
     TimelineMetricMetadata value1 = new TimelineMetricMetadata("dummy_metric1",
-      "dummy_app1", "Integer", null, 1L, true);
+      "dummy_app1", "Integer", null, 1L, true, false);
     TimelineMetricMetadata value2 = new TimelineMetricMetadata("dummy_metric2",
-      "dummy_app2", "Integer", null, 1L, true);
+      "dummy_app2", "Integer", null, 1L, true, false);
+    TimelineMetricMetadata value3 = new TimelineMetricMetadata("dummy_metric3",
+      "dummy_app3", "Integer", null, 1L, true, true);
 
     Assert.assertEquals(value1, cachedData.get(key1));
     Assert.assertEquals(value2, cachedData.get(key2));
+    Assert.assertEquals(value3, cachedData.get(key3));
 
     TimelineMetricMetadataSync syncRunnable = new TimelineMetricMetadataSync(metadataManager);
     syncRunnable.run();
@@ -117,16 +120,19 @@ public class TestMetadataManager extends AbstractMiniHBaseClusterTest {
       hdb.getTimelineMetricMetadata();
 
     Assert.assertNotNull(savedData);
-    Assert.assertEquals(2, savedData.size());
+    Assert.assertEquals(3, savedData.size());
     Assert.assertEquals(value1, savedData.get(key1));
     Assert.assertEquals(value2, savedData.get(key2));
+    Assert.assertEquals(value3, savedData.get(key3));
 
     Map<String, Set<String>> cachedHostData = metadataManager.getHostedAppsCache();
     Map<String, Set<String>> savedHostData = metadataManager.getHostedAppsFromStore();
     Assert.assertEquals(cachedData.size(), savedData.size());
     Assert.assertEquals("dummy_app1", cachedHostData.get("dummy_host1").iterator().next());
     Assert.assertEquals("dummy_app2", cachedHostData.get("dummy_host2").iterator().next());
+    Assert.assertEquals("dummy_app3", cachedHostData.get("dummy_host3").iterator().next());
     Assert.assertEquals("dummy_app1", savedHostData.get("dummy_host1").iterator().next());
     Assert.assertEquals("dummy_app2", savedHostData.get("dummy_host2").iterator().next());
+    Assert.assertEquals("dummy_app3", cachedHostData.get("dummy_host3").iterator().next());
   }
 }

ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataSync.java

@@ -41,9 +41,9 @@ public class TestMetadataSync {
     PhoenixHBaseAccessor hBaseAccessor = createNiceMock(PhoenixHBaseAccessor.class);
 
     final TimelineMetricMetadata testMetadata1 = new TimelineMetricMetadata(
-      "m1", "a1", "", GAUGE.name(), System.currentTimeMillis(), true);
+      "m1", "a1", "", GAUGE.name(), System.currentTimeMillis(), true, false);
     final TimelineMetricMetadata testMetadata2 = new TimelineMetricMetadata(
-      "m2", "a2", "", GAUGE.name(), System.currentTimeMillis(), true);
+      "m2", "a2", "", GAUGE.name(), System.currentTimeMillis(), true, false);
 
     Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata =
       new HashMap<TimelineMetricMetadataKey, TimelineMetricMetadata>() {{
@@ -88,11 +88,11 @@ public class TestMetadataSync {
     PhoenixHBaseAccessor hBaseAccessor = createNiceMock(PhoenixHBaseAccessor.class);
 
     TimelineMetricMetadata metadata1 = new TimelineMetricMetadata(
-      "xxx.abc.yyy", "a1", "", GAUGE.name(), System.currentTimeMillis(), true);
+      "xxx.abc.yyy", "a1", "", GAUGE.name(), System.currentTimeMillis(), true, false);
     TimelineMetricMetadata metadata2 = new TimelineMetricMetadata(
-      "xxx.cdef.yyy", "a2", "", GAUGE.name(), System.currentTimeMillis(), true);
+      "xxx.cdef.yyy", "a2", "", GAUGE.name(), System.currentTimeMillis(), true, false);
     TimelineMetricMetadata metadata3 = new TimelineMetricMetadata(
-      "xxx.pqr.zzz", "a3", "", GAUGE.name(), System.currentTimeMillis(), true);
+      "xxx.pqr.zzz", "a3", "", GAUGE.name(), System.currentTimeMillis(), true, false);
 
     expect(configuration.get(TIMELINE_METRIC_METADATA_FILTERS)).andReturn("abc,cde");
 

ambari-metrics/ambari-metrics-timelineservice/src/test/resources/test_data/metric_whitelist.dat

@@ -0,0 +1,6 @@
+cpu_system
+cpu_user
+mem_cached
+mem_free
+._p_dfs.FSNamesystem.*
+._p_jvm.JvmMetrics*