AMBARI-18125 : Allow for certain metrics to skip aggregation determined by client. (avijayan)

ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.metrics2.sink.timeline;
 
+import java.util.HashMap;
 import java.util.Map;
 import java.util.TreeMap;
 
@@ -44,6 +45,7 @@ public class TimelineMetric implements Comparable<TimelineMetric> {
   private String type;
   private String units;
   private TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
+  private Map<String, String> metadata = new HashMap<>();
 
   // default
   public TimelineMetric() {
@@ -148,6 +150,15 @@ public class TimelineMetric implements Comparable<TimelineMetric> {
     this.metricValues.putAll(metricValues);
   }
 
+  @XmlElement(name = "metadata")
+  public Map<String,String> getMetadata () {
+    return metadata;
+  }
+
+  public void setMetadata (Map<String,String> metadata) {
+    this.metadata = metadata;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) return true;

ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java

@@ -39,6 +39,7 @@ import java.net.SocketAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -205,6 +206,7 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple
       String contextName = record.context();
 
       StringBuilder sb = new StringBuilder();
+      boolean skipAggregation = false;
 
       // Transform ipc.8020 -> ipc.client,  ipc.8040 -> ipc.datanode, etc.
       if (contextName.startsWith("ipc.")) {
@@ -216,17 +218,19 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple
 
       sb.append(contextName);
       sb.append('.');
-      // Similar to GangliaContext adding processName to distinguish jvm
-      // metrics for co-hosted daemons. We only do this for HBase since the
-      // appId is shared for Master and RS.
-      if (contextName.equals("jvm")) {
-        if (record.tags() != null) {
-          for (MetricsTag tag : record.tags()) {
-            if (tag.info().name().equalsIgnoreCase("processName") &&
-               (tag.value().equals("RegionServer") || tag.value().equals("Master"))) {
-              sb.append(tag.value());
-              sb.append('.');
-            }
+
+      if (record.tags() != null) {
+        for (MetricsTag tag : record.tags()) {
+          if (StringUtils.isNotEmpty(tag.name()) && tag.name().equals("skipAggregation")) {
+            skipAggregation = String.valueOf(true).equals(tag.value());
+          }
+                // Similar to GangliaContext adding processName to distinguish jvm
+                // metrics for co-hosted daemons. We only do this for HBase since the
+                // appId is shared for Master and RS.
+          if (contextName.equals("jvm") && tag.info().name().equalsIgnoreCase("processName") &&
+            (tag.value().equals("RegionServer") || tag.value().equals("Master"))) {
+            sb.append(tag.value());
+            sb.append('.');
           }
         }
       }
@@ -258,6 +262,12 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple
       Collection<AbstractMetric> metrics = (Collection<AbstractMetric>) record.metrics();
 
       List<TimelineMetric> metricList = new ArrayList<TimelineMetric>();
+      Map<String, String> metadata = null;
+      if (skipAggregation) {
+        metadata = Collections.singletonMap("skipAggregation", "true");
+      }
+
+
       long startTime = record.timestamp();
 
       for (AbstractMetric metric : metrics) {
@@ -271,6 +281,9 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple
         timelineMetric.setStartTime(startTime);
         timelineMetric.setType(metric.type() != null ? metric.type().name() : null);
         timelineMetric.getMetricValues().put(startTime, value.doubleValue());
+        if (metadata != null) {
+          timelineMetric.setMetadata(metadata);
+        }
         // Put intermediate values into the cache until it is time to send
         boolean isCounter = MetricType.COUNTER == metric.type();
         metricsCache.putTimelineMetric(timelineMetric, isCounter);

ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java

@@ -324,8 +324,8 @@ public class HadoopTimelineMetricsSinkTest {
         }
       }, "8040")
     );
-    expect(record.tags()).andReturn(tags1).times(6);
-    expect(record.tags()).andReturn(tags2).times(6);
+    expect(record.tags()).andReturn(tags1).times(12);
+    expect(record.tags()).andReturn(tags2).times(12);
 
     sink.appendPrefix(eq(record), (StringBuilder) anyObject());
     expectLastCall().anyTimes().andStubAnswer(new IAnswer<Object>() {

ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py

@@ -248,8 +248,10 @@ class HostInfo():
 
     skip_disk_patterns = self.__config.get_disk_metrics_skip_pattern()
     logger.debug('skip_disk_patterns: %s' % skip_disk_patterns)
+    print skip_disk_patterns
     if not skip_disk_patterns or skip_disk_patterns == 'None':
       io_counters = psutil.disk_io_counters()
+      print io_counters
     else:
       sdiskio = namedtuple('sdiskio', ['read_count', 'write_count',
                                        'read_bytes', 'write_bytes',
@@ -303,7 +305,7 @@ class HostInfo():
         disk = item[0]
         logger.debug('Adding disk counters for %s' % str(disk))
         sdiskio = item[1]
-        prefix = 'disk_{0}_'.format(disk_counter)
+        prefix = 'sdisk_{0}_'.format(disk)
         counter_dict = {
           prefix + 'read_count' : sdiskio.read_count if hasattr(sdiskio, 'read_count') else 0,
           prefix + 'write_count' : sdiskio.write_count if hasattr(sdiskio, 'write_count') else 0,

ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestHostInfo.py

@@ -23,6 +23,7 @@ from host_info import HostInfo
 import platform
 from unittest import TestCase
 from mock.mock import patch, MagicMock
+from core.config_reader import Configuration
 import collections
 
 logger = logging.getLogger()
@@ -138,7 +139,9 @@ class TestHostInfo(TestCase):
                                                   'read_time', 'write_time'])
     io_mock.return_value = Counters(0, 1, 2, 3, 4, 5)
 
-    hostinfo = HostInfo(MagicMock())
+    c = MagicMock()
+    c.get_disk_metrics_skip_pattern.return_value = None
+    hostinfo = HostInfo(c)
 
     disk_counters = hostinfo.get_combined_disk_io_counters()
 
@@ -179,28 +182,28 @@ class TestHostInfo(TestCase):
 
     disk_counter_per_disk = hostinfo.get_disk_io_counters_per_disk()
 
-    # Assert for sda1
-    self.assertEqual(disk_counter_per_disk['disk_1_read_count'], 0)
-    self.assertEqual(disk_counter_per_disk['disk_1_write_count'], 1)
-    self.assertEqual(disk_counter_per_disk['disk_1_read_bytes'], 2)
-    self.assertEqual(disk_counter_per_disk['disk_1_write_bytes'], 3)
-    self.assertEqual(disk_counter_per_disk['disk_1_read_time'], 4)
-    self.assertEqual(disk_counter_per_disk['disk_1_write_time'], 5)
-    self.assertEqual(disk_counter_per_disk['disk_1_busy_time'], 6)
-    self.assertEqual(disk_counter_per_disk['disk_1_read_merged_count'], 7)
-    self.assertEqual(disk_counter_per_disk['disk_1_write_merged_count'], 8)
+    # Assert for sdisk_sda1
+    self.assertEqual(disk_counter_per_disk['sdisk_sda1_read_count'], 0)
+    self.assertEqual(disk_counter_per_disk['sdisk_sda1_write_count'], 1)
+    self.assertEqual(disk_counter_per_disk['sdisk_sda1_read_bytes'], 2)
+    self.assertEqual(disk_counter_per_disk['sdisk_sda1_write_bytes'], 3)
+    self.assertEqual(disk_counter_per_disk['sdisk_sda1_read_time'], 4)
+    self.assertEqual(disk_counter_per_disk['sdisk_sda1_write_time'], 5)
+    self.assertEqual(disk_counter_per_disk['sdisk_sda1_busy_time'], 6)
+    self.assertEqual(disk_counter_per_disk['sdisk_sda1_read_merged_count'], 7)
+    self.assertEqual(disk_counter_per_disk['sdisk_sda1_write_merged_count'], 8)
 
     # Assert for sdb1
 
-    self.assertEqual(disk_counter_per_disk['disk_2_read_count'], 9)
-    self.assertEqual(disk_counter_per_disk['disk_2_write_count'], 10)
-    self.assertEqual(disk_counter_per_disk['disk_2_read_bytes'], 11)
-    self.assertEqual(disk_counter_per_disk['disk_2_write_bytes'], 12)
-    self.assertEqual(disk_counter_per_disk['disk_2_read_time'], 13)
-    self.assertEqual(disk_counter_per_disk['disk_2_write_time'], 14)
-    self.assertEqual(disk_counter_per_disk['disk_2_busy_time'], 15)
-    self.assertEqual(disk_counter_per_disk['disk_2_read_merged_count'], 16)
-    self.assertEqual(disk_counter_per_disk['disk_2_write_merged_count'], 17)
+    self.assertEqual(disk_counter_per_disk['sdisk_sdb1_read_count'], 9)
+    self.assertEqual(disk_counter_per_disk['sdisk_sdb1_write_count'], 10)
+    self.assertEqual(disk_counter_per_disk['sdisk_sdb1_read_bytes'], 11)
+    self.assertEqual(disk_counter_per_disk['sdisk_sdb1_write_bytes'], 12)
+    self.assertEqual(disk_counter_per_disk['sdisk_sdb1_read_time'], 13)
+    self.assertEqual(disk_counter_per_disk['sdisk_sdb1_write_time'], 14)
+    self.assertEqual(disk_counter_per_disk['sdisk_sdb1_busy_time'], 15)
+    self.assertEqual(disk_counter_per_disk['sdisk_sdb1_read_merged_count'], 16)
+    self.assertEqual(disk_counter_per_disk['sdisk_sdb1_write_merged_count'], 17)
 
 
 

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java

@@ -246,6 +246,9 @@ public class TimelineMetricConfiguration {
   public static final String DEFAULT_TOPN_HOSTS_LIMIT =
     "timeline.metrics.default.topn.hosts.limit";
 
+  public static final String TIMELINE_METRIC_AGGREGATION_SQL_FILTERS =
+    "timeline.metrics.cluster.aggregation.sql.filters";
+
   public static final String HOST_APP_ID = "HOST";
 
   private Configuration hbaseConf;

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java

@@ -23,7 +23,9 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.metrics2.sink.timeline.PostProcessingUtil;
 import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataKey;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.DefaultCondition;
@@ -31,19 +33,18 @@ import java.io.IOException;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.ArrayList;
-import java.util.Collections;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HOST_APP_ID;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.SERVER_SIDE_TIMESIFT_ADJUSTMENT;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_METRIC_SQL;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_AGGREGATION_SQL_FILTERS;
 
 /**
  * Aggregates a metric across all hosts in the cluster. Reads metrics from
@@ -57,6 +58,8 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
   // 1 minute client side buffering adjustment
   private final Long serverTimeShiftAdjustment;
   private final boolean interpolationEnabled;
+  private TimelineMetricMetadataManager metadataManagerInstance;
+  private String skipAggrPatternStrings;
 
 
   public TimelineMetricClusterAggregatorSecond(String aggregatorName,
@@ -75,10 +78,12 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
       sleepIntervalMillis, checkpointCutOffMultiplier, aggregatorDisabledParam,
       tableName, outputTableName, nativeTimeRangeDelay);
 
+    this.metadataManagerInstance = metadataManager;
     appAggregator = new TimelineMetricAppAggregator(metadataManager, metricsConf);
     this.timeSliceIntervalMillis = timeSliceInterval;
     this.serverTimeShiftAdjustment = Long.parseLong(metricsConf.get(SERVER_SIDE_TIMESIFT_ADJUSTMENT, "90000"));
     this.interpolationEnabled = Boolean.parseBoolean(metricsConf.get(TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED, "true"));
+    this.skipAggrPatternStrings = metricsConf.get(TIMELINE_METRIC_AGGREGATION_SQL_FILTERS);
   }
 
   @Override
@@ -100,8 +105,19 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
 
   @Override
   protected Condition prepareMetricQueryCondition(long startTime, long endTime) {
-    Condition condition = new DefaultCondition(null, null, null, null, startTime - serverTimeShiftAdjustment,
+
+    List<String> metricNames = new ArrayList<>();
+    boolean metricNamesNotCondition = false;
+
+    if (!StringUtils.isEmpty(skipAggrPatternStrings)) {
+      LOG.info("Skipping aggregation for metric patterns : " + skipAggrPatternStrings);
+      metricNames.addAll(Arrays.asList(skipAggrPatternStrings.split(",")));
+      metricNamesNotCondition = true;
+    }
+
+    Condition condition = new DefaultCondition(metricNames, null, null, null, startTime - serverTimeShiftAdjustment,
       endTime, null, null, true);
+    condition.setMetricNamesNotCondition(metricNamesNotCondition);
     condition.setNoLimit();
     condition.setFetchSize(resultsetFetchSize);
     condition.setStatement(String.format(GET_METRIC_SQL,
@@ -177,6 +193,15 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
   protected int processAggregateClusterMetrics(Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics,
                                               TimelineMetric metric, List<Long[]> timeSlices) {
     // Create time slices
+
+    TimelineMetricMetadataKey appKey =  new TimelineMetricMetadataKey(metric.getMetricName(), metric.getAppId());
+    TimelineMetricMetadata metricMetadata = metadataManagerInstance.getMetadataCacheValue(appKey);
+
+    if (metricMetadata != null && !metricMetadata.isSupportsAggregates()) {
+      LOG.debug("Skipping cluster aggregation for " + metric.getMetricName());
+      return 0;
+    }
+
     Map<TimelineClusterMetric, Double> clusterMetrics = sliceFromTimelineMetric(metric, timeSlices);
     int numHosts = 0;
 

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery;
 
+import org.apache.commons.collections.MapUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -24,7 +25,6 @@ import org.apache.hadoop.metrics2.sink.timeline.MetadataException;
 import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
 import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineClusterMetric;
 
 import java.sql.SQLException;
 import java.util.Collection;
@@ -172,7 +172,7 @@ public class TimelineMetricMetadataManager {
       timelineMetric.getUnits(),
       timelineMetric.getType(),
       timelineMetric.getStartTime(),
-      true
+      supportAggregates(timelineMetric)
     );
   }
 
@@ -187,4 +187,9 @@ public class TimelineMetricMetadataManager {
   public boolean isDisabled() {
     return isDisabled;
   }
+
+  private boolean supportAggregates(TimelineMetric metric) {
+    return MapUtils.isEmpty(metric.getMetadata()) ||
+      !(String.valueOf(true).equals(metric.getMetadata().get("skipAggregation")));
+  }
 }

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/Condition.java

@@ -44,4 +44,5 @@ public interface Condition {
   void addOrderByColumn(String column);
   void setNoLimit();
   boolean doUpdate();
+  void setMetricNamesNotCondition(boolean metricNamesNotCondition);
 }

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultCondition.java

@@ -16,12 +16,14 @@
  * limitations under the License.
  */
 package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.metrics2.sink.timeline.TopNConfig;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
 import org.apache.hadoop.metrics2.sink.timeline.Precision;
 
+import java.util.ArrayList;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
@@ -40,6 +42,7 @@ public class DefaultCondition implements Condition {
   Integer fetchSize;
   String statement;
   Set<String> orderByColumns = new LinkedHashSet<String>();
+  boolean metricNamesNotCondition = false;
 
   private static final Log LOG = LogFactory.getLog(DefaultCondition.class);
 
@@ -215,44 +218,85 @@ public class DefaultCondition implements Condition {
 
   protected boolean appendMetricNameClause(StringBuilder sb) {
     boolean appendConjunction = false;
-    StringBuilder metricsLike = new StringBuilder();
-    StringBuilder metricsIn = new StringBuilder();
+    List<String> metricsLike = new ArrayList<>();
+    List<String> metricsIn = new ArrayList<>();
 
     if (getMetricNames() != null) {
       for (String name : getMetricNames()) {
         if (name.contains("%")) {
-          if (metricsLike.length() > 1) {
-            metricsLike.append(" OR ");
-          }
-          metricsLike.append("METRIC_NAME LIKE ?");
+          metricsLike.add(name);
         } else {
-          if (metricsIn.length() > 0) {
-            metricsIn.append(", ");
-          }
-          metricsIn.append("?");
+          metricsIn.add(name);
         }
       }
 
-      if (metricsIn.length() > 0) {
-        sb.append("(METRIC_NAME IN (");
-        sb.append(metricsIn);
+      // Put a '(' first
+      sb.append("(");
+
+      //IN clause
+      // METRIC_NAME (NOT) IN (?,?,?,?)
+      if (CollectionUtils.isNotEmpty(metricsIn)) {
+        sb.append("METRIC_NAME");
+        if (metricNamesNotCondition) {
+          sb.append(" NOT");
+        }
+        sb.append(" IN (");
+        //Append ?,?,?,?
+        for (int i = 0; i < metricsIn.size(); i++) {
+          sb.append("?");
+          if (i < metricsIn.size() - 1) {
+            sb.append(", ");
+          }
+        }
         sb.append(")");
         appendConjunction = true;
       }
 
-      if (metricsLike.length() > 0) {
-        if (appendConjunction) {
-          sb.append(" OR ");
+      //Put an OR/AND if both types are present
+      if (CollectionUtils.isNotEmpty(metricsIn) &&
+        CollectionUtils.isNotEmpty(metricsLike)) {
+        if (metricNamesNotCondition) {
+          sb.append(" AND ");
         } else {
-          sb.append("(");
+          sb.append(" OR ");
+        }
+      }
+
+      //LIKE clause
+      // METRIC_NAME (NOT) LIKE ? OR(AND) METRIC_NAME LIKE ?
+      if (CollectionUtils.isNotEmpty(metricsLike)) {
+
+        for (int i = 0; i < metricsLike.size(); i++) {
+          sb.append("METRIC_NAME");
+          if (metricNamesNotCondition) {
+            sb.append(" NOT");
+          }
+          sb.append(" LIKE ");
+          sb.append("?");
+
+          if (i < metricsLike.size() - 1) {
+            if (metricNamesNotCondition) {
+              sb.append(" AND ");
+            } else {
+              sb.append(" OR ");
+            }
+          }
         }
-        sb.append(metricsLike);
         appendConjunction = true;
       }
 
+      // Finish with a ')'
       if (appendConjunction) {
         sb.append(")");
       }
+
+      metricNames.clear();
+      if (CollectionUtils.isNotEmpty(metricsIn)) {
+        metricNames.addAll(metricsIn);
+      }
+      if (CollectionUtils.isNotEmpty(metricsLike)) {
+        metricNames.addAll(metricsLike);
+      }
     }
     return appendConjunction;
   }
@@ -333,4 +377,8 @@ public class DefaultCondition implements Condition {
     }
     return false;
   }
+
+  public void setMetricNamesNotCondition(boolean metricNamesNotCondition) {
+    this.metricNamesNotCondition = metricNamesNotCondition;
+  }
 }

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/EmptyCondition.java

@@ -27,6 +27,7 @@ import java.util.List;
 public class EmptyCondition implements Condition {
   String statement;
   boolean doUpdate = false;
+  boolean metricNamesNotCondition = false;
 
   @Override
   public boolean isEmpty() {
@@ -144,4 +145,9 @@ public class EmptyCondition implements Condition {
       " doUpdate = " + this.doUpdate() +
       " }";
   }
+
+  @Override
+  public void setMetricNamesNotCondition(boolean metricNamesNotCondition) {
+    this.metricNamesNotCondition = metricNamesNotCondition;
+  }
 }

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/SplitByMetricNamesCondition.java

@@ -25,6 +25,7 @@ import java.util.List;
 public class SplitByMetricNamesCondition implements Condition {
   private final Condition adaptee;
   private String currentMetric;
+  private boolean metricNamesNotCondition = false;
 
   public SplitByMetricNamesCondition(Condition condition){
     this.adaptee = condition;
@@ -180,4 +181,9 @@ public class SplitByMetricNamesCondition implements Condition {
   public void setCurrentMetric(String currentMetric) {
     this.currentMetric = currentMetric;
   }
+
+ @Override
+  public void setMetricNamesNotCondition(boolean metricNamesNotCondition) {
+    this.metricNamesNotCondition = metricNamesNotCondition;
+  }
 }

ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java

@@ -47,7 +47,7 @@ public class TestPhoenixTransactSQL {
   @Test
   public void testConditionClause() throws Exception {
     Condition condition = new DefaultCondition(
-      Arrays.asList("cpu_user", "mem_free"), Collections.singletonList("h1"),
+      new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
       "a1", "i1", 1407959718L, 1407959918L, null, null, false);
 
     String preparedClause = condition.getConditionClause().toString();
@@ -78,7 +78,7 @@ public class TestPhoenixTransactSQL {
   @Test
   public void testLikeConditionClause() throws Exception {
     Condition condition = new DefaultCondition(
-      Arrays.asList("cpu_user", "some=%.metric"),
+      new ArrayList<>(Arrays.asList("cpu_user", "some=%.metric")),
       Collections.singletonList("h1"), "a1", "i1", 1407959718L, 1407959918L,
       null, null, false);
 
@@ -115,7 +115,7 @@ public class TestPhoenixTransactSQL {
 
 
     condition = new DefaultCondition(
-        Arrays.asList("some=%.metric"), Collections.singletonList("h1"), "a1", "i1",
+      new ArrayList<>(Arrays.asList("some=%.metric")), Collections.singletonList("h1"), "a1", "i1",
         1407959718L, 1407959918L, null, null, false);
 
     preparedClause = condition.getConditionClause().toString();
@@ -127,7 +127,7 @@ public class TestPhoenixTransactSQL {
 
 
     condition = new DefaultCondition(
-      Arrays.asList("some=%.metric1", "some=%.metric2", "some=%.metric3"),
+      new ArrayList<>(Arrays.asList("some=%.metric1", "some=%.metric2", "some=%.metric3")),
       Collections.singletonList("h1"), "a1", "i1",
       1407959718L, 1407959918L, null, null, false);
 
@@ -142,7 +142,7 @@ public class TestPhoenixTransactSQL {
   @Test
   public void testPrepareGetAggregatePrecisionMINUTES() throws SQLException {
     Condition condition = new DefaultCondition(
-      Arrays.asList("cpu_user", "mem_free"), Collections.singletonList("h1"),
+      new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
       "a1", "i1", 1407959718L, 1407959918L, Precision.MINUTES, null, false);
     Connection connection = createNiceMock(Connection.class);
     PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
@@ -163,7 +163,7 @@ public class TestPhoenixTransactSQL {
     Long startTime = 1407959718L;
     //SECONDS precision
     Condition condition = new DefaultCondition(
-      Arrays.asList("cpu_user", "mem_free"), Collections.singletonList("h1"),
+      new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
       "a1", "i1", startTime, endTime, null, null, false);
     Connection connection = createNiceMock(Connection.class);
     PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
@@ -181,7 +181,7 @@ public class TestPhoenixTransactSQL {
     // MINUTES precision
     startTime = endTime-PhoenixTransactSQL.DAY/1000;
     condition = new DefaultCondition(
-      Arrays.asList("cpu_user", "mem_free"), Collections.singletonList("h1"),
+      new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
       "a1", "i1", startTime, endTime, null, null, false);
     connection = createNiceMock(Connection.class);
     preparedStatement = createNiceMock(PreparedStatement.class);
@@ -199,7 +199,7 @@ public class TestPhoenixTransactSQL {
     // HOURS precision
     startTime = endTime-PhoenixTransactSQL.DAY*30/1000;
     condition = new DefaultCondition(
-      Arrays.asList("cpu_user", "mem_free"), Collections.singletonList("h1"),
+      new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
       "a1", "i1", startTime, endTime, null, null, false);
     connection = createNiceMock(Connection.class);
     preparedStatement = createNiceMock(PreparedStatement.class);
@@ -217,7 +217,7 @@ public class TestPhoenixTransactSQL {
     // DAYS precision
     startTime = endTime-PhoenixTransactSQL.DAY*30*2/1000;
     condition = new DefaultCondition(
-      Arrays.asList("cpu_user", "mem_free"), Collections.singletonList("h1"),
+      new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
       "a1", "i1", startTime, endTime, null, null, false);
     connection = createNiceMock(Connection.class);
     preparedStatement = createNiceMock(PreparedStatement.class);
@@ -236,7 +236,7 @@ public class TestPhoenixTransactSQL {
   @Test
   public void testPrepareGetAggregatePrecisionHours() throws SQLException {
     Condition condition = new DefaultCondition(
-      Arrays.asList("cpu_user", "mem_free"), Collections.singletonList("h1"),
+      new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
       "a1", "i1", 1407959718L, 1407959918L, Precision.HOURS, null, false);
     Connection connection = createNiceMock(Connection.class);
     PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
@@ -254,7 +254,7 @@ public class TestPhoenixTransactSQL {
   @Test
   public void testPrepareGetMetricsPrecisionMinutes() throws SQLException {
     Condition condition = new DefaultCondition(
-      Arrays.asList("cpu_user", "mem_free"), Collections.singletonList("h1"),
+      new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
       "a1", "i1", 1407959718L, 1407959918L, Precision.MINUTES, null, false);
     Connection connection = createNiceMock(Connection.class);
     PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
@@ -275,7 +275,7 @@ public class TestPhoenixTransactSQL {
     Long startTime = endTime - 200;
     // SECONDS precision
     Condition condition = new DefaultCondition(
-      Arrays.asList("cpu_user", "mem_free"), Collections.singletonList("h1"),
+      new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
       "a1", "i1", startTime, endTime, null, null, false);
     Connection connection = createNiceMock(Connection.class);
     PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
@@ -294,7 +294,7 @@ public class TestPhoenixTransactSQL {
     // SECONDS precision
     startTime = endTime-PhoenixTransactSQL.HOUR*2/1000;
     condition = new DefaultCondition(
-      Arrays.asList("cpu_user", "mem_free"), Collections.singletonList("h1"),
+      new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
       "a1", "i1", startTime, endTime, null, null, false);
     connection = createNiceMock(Connection.class);
     preparedStatement = createNiceMock(PreparedStatement.class);
@@ -311,7 +311,7 @@ public class TestPhoenixTransactSQL {
     // MINUTES precision
     startTime = endTime-PhoenixTransactSQL.DAY/1000;
     condition = new DefaultCondition(
-      Arrays.asList("cpu_user", "mem_free"), Collections.singletonList("h1"),
+      new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
       "a1", "i1", startTime, endTime, null, null, false);
     connection = createNiceMock(Connection.class);
     preparedStatement = createNiceMock(PreparedStatement.class);
@@ -328,7 +328,7 @@ public class TestPhoenixTransactSQL {
     // HOURS precision
     startTime = endTime-PhoenixTransactSQL.DAY*30/1000;
     condition = new DefaultCondition(
-      Arrays.asList("cpu_user", "mem_free"), Collections.singletonList("h1"),
+      new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
       "a1", "i1", startTime, endTime, null, null, false);
     connection = createNiceMock(Connection.class);
     preparedStatement = createNiceMock(PreparedStatement.class);
@@ -345,7 +345,7 @@ public class TestPhoenixTransactSQL {
     // DAYS precision
     startTime = endTime-PhoenixTransactSQL.DAY*30*2/1000;
     condition = new DefaultCondition(
-      Arrays.asList("cpu_user", "mem_free"), Collections.singletonList("h1"),
+      new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
       "a1", "i1", startTime, endTime, null, null, false);
     connection = createNiceMock(Connection.class);
     preparedStatement = createNiceMock(PreparedStatement.class);
@@ -364,7 +364,7 @@ public class TestPhoenixTransactSQL {
   @Test
   public void testPrepareGetLatestMetricSqlStmtMultipleHostNames() throws SQLException {
     Condition condition = new DefaultCondition(
-      Arrays.asList("cpu_user", "mem_free"), Arrays.asList("h1", "h2"),
+      new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Arrays.asList("h1", "h2"),
       "a1", "i1", null, null, null, null, false);
     Connection connection = createNiceMock(Connection.class);
     PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
@@ -390,7 +390,7 @@ public class TestPhoenixTransactSQL {
   public void testPrepareGetLatestMetricSqlStmtSortMergeJoinAlgorithm()
     throws SQLException {
     Condition condition = new DefaultCondition(
-      Arrays.asList("cpu_user", "mem_free"), Arrays.asList("h1"),
+      new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Arrays.asList("h1"),
       "a1", "i1", null, null, null, null, false);
     Connection connection = createNiceMock(Connection.class);
     PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
@@ -413,7 +413,7 @@ public class TestPhoenixTransactSQL {
   @Test
   public void testPrepareGetMetricsPrecisionHours() throws SQLException {
     Condition condition = new DefaultCondition(
-      Arrays.asList("cpu_user", "mem_free"), Collections.singletonList("h1"),
+      new ArrayList<>(Arrays.asList("cpu_user", "mem_free")), Collections.singletonList("h1"),
       "a1", "i1", 1407959718L, 1407959918L, Precision.HOURS, null, false);
     Connection connection = createNiceMock(Connection.class);
     PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
@@ -561,7 +561,7 @@ public class TestPhoenixTransactSQL {
     List<String> hosts = Arrays.asList("h1", "h2", "h3", "h4");
 
     Condition condition = new TopNCondition(
-      Arrays.asList("cpu_user"), hosts,
+      new ArrayList<>(Collections.singletonList("cpu_user")), hosts,
       "a1", "i1", 1407959718L, 1407959918L, null, null, false, 2, null, false);
 
     String conditionClause = condition.getConditionClause().toString();
@@ -580,7 +580,7 @@ public class TestPhoenixTransactSQL {
 
   @Test
   public void testTopNMetricsConditionClause() throws Exception {
-    List<String> metricNames = Arrays.asList("m1", "m2", "m3");
+    List<String> metricNames = new ArrayList<>(Arrays.asList("m1", "m2", "m3"));
 
     Condition condition = new TopNCondition(
       metricNames, Collections.singletonList("h1"),
@@ -602,7 +602,7 @@ public class TestPhoenixTransactSQL {
 
   @Test
   public void testTopNMetricsIllegalConditionClause() throws Exception {
-    List<String> metricNames = Arrays.asList("m1", "m2");
+    List<String> metricNames = new ArrayList<>(Arrays.asList("m1", "m2"));
 
     List<String> hosts = Arrays.asList("h1", "h2");
 
@@ -616,7 +616,7 @@ public class TestPhoenixTransactSQL {
   @Test
   public void testHostsRegexpConditionClause() {
     Condition condition = new TopNCondition(
-            Arrays.asList("m1"), Arrays.asList("%.ambari", "host1.apache"),
+      new ArrayList<>(Arrays.asList("m1")), Arrays.asList("%.ambari", "host1.apache"),
             "a1", "i1", 1407959718L, 1407959918L, null, null, false, 2, null, false);
 
     String conditionClause = condition.getConditionClause().toString();
@@ -629,7 +629,7 @@ public class TestPhoenixTransactSQL {
     Assert.assertEquals(expectedClause, conditionClause);
 
     condition = new TopNCondition(
-            Arrays.asList("m1"), Arrays.asList("%.ambari"),
+      new ArrayList<>(Arrays.asList("m1")), Arrays.asList("%.ambari"),
             "a1", "i1", 1407959718L, 1407959918L, null, null, false, 2, null, false);
 
     conditionClause = condition.getConditionClause().toString();
@@ -643,7 +643,7 @@ public class TestPhoenixTransactSQL {
     Assert.assertEquals(expectedClause, conditionClause);
 
     condition = new TopNCondition(
-            Arrays.asList("m1", "m2", "m3"), Arrays.asList("h1.ambari"),
+      new ArrayList<>(Arrays.asList("m1", "m2", "m3")), Arrays.asList("h1.ambari"),
             "a1", "i1", 1407959718L, 1407959918L, null, null, false, 2, null, false);
 
     conditionClause = condition.getConditionClause().toString();

ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecondTest.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline
 import junit.framework.Assert;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataKey;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager;
 import org.easymock.EasyMock;
 import org.junit.Test;
@@ -114,6 +115,10 @@ public class TimelineMetricClusterAggregatorSecondTest {
     Configuration configuration = new Configuration();
     TimelineMetricMetadataManager metricMetadataManagerMock = EasyMock.createNiceMock(TimelineMetricMetadataManager.class);
 
+    EasyMock.expect(metricMetadataManagerMock.getMetadataCacheValue((TimelineMetricMetadataKey)EasyMock.anyObject()))
+      .andReturn(null).anyTimes();
+    EasyMock.replay(metricMetadataManagerMock);
+
     TimelineMetricClusterAggregatorSecond secondAggregator = new TimelineMetricClusterAggregatorSecond(
       "TimelineClusterAggregatorSecond", metricMetadataManagerMock, null, configuration, null,
       aggregatorInterval, 2, "false", "", "", aggregatorInterval, sliceInterval

ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/DefaultConditionTest.java

@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query;
+
+import junit.framework.Assert;
+import org.apache.commons.collections.CollectionUtils;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DefaultConditionTest {
+
+  @Test
+  public void testMetricNameWhereCondition() {
+    List<String> metricNames = new ArrayList<>();
+
+    //Only IN clause.
+
+    metricNames.add("M1");
+    DefaultCondition condition = new DefaultCondition(metricNames,null,null,null,null,null,null,null,true);
+    StringBuilder sb = new StringBuilder();
+    condition.appendMetricNameClause(sb);
+    Assert.assertEquals(sb.toString(), "(METRIC_NAME IN (?))");
+    Assert.assertTrue(CollectionUtils.isEqualCollection(metricNames, condition.getMetricNames()));
+
+    metricNames.add("m2");
+    condition = new DefaultCondition(metricNames,null,null,null,null,null,null,null,true);
+    sb = new StringBuilder();
+    condition.appendMetricNameClause(sb);
+    Assert.assertEquals(sb.toString(), "(METRIC_NAME IN (?, ?))");
+    Assert.assertTrue(CollectionUtils.isEqualCollection(metricNames, condition.getMetricNames()));
+
+    // Only NOT IN clause
+    condition = new DefaultCondition(metricNames,null,null,null,null,null,null,null,true);
+    condition.setMetricNamesNotCondition(true);
+    sb = new StringBuilder();
+    condition.appendMetricNameClause(sb);
+    Assert.assertEquals(sb.toString(), "(METRIC_NAME NOT IN (?, ?))");
+    Assert.assertTrue(CollectionUtils.isEqualCollection(metricNames, condition.getMetricNames()));
+
+    metricNames.clear();
+
+    //Only LIKE clause
+    metricNames.add("disk%");
+    condition = new DefaultCondition(metricNames,null,null,null,null,null,null,null,true);
+    sb = new StringBuilder();
+    condition.appendMetricNameClause(sb);
+    Assert.assertEquals(sb.toString(), "(METRIC_NAME LIKE ?)");
+    Assert.assertTrue(CollectionUtils.isEqualCollection(metricNames, condition.getMetricNames()));
+
+    metricNames.add("cpu%");
+    condition = new DefaultCondition(metricNames,null,null,null,null,null,null,null,true);
+    sb = new StringBuilder();
+    condition.appendMetricNameClause(sb);
+    Assert.assertEquals(sb.toString(), "(METRIC_NAME LIKE ? OR METRIC_NAME LIKE ?)");
+    Assert.assertTrue(CollectionUtils.isEqualCollection(metricNames, condition.getMetricNames()));
+
+    //Only NOT LIKE clause
+    condition = new DefaultCondition(metricNames,null,null,null,null,null,null,null,true);
+    condition.setMetricNamesNotCondition(true);
+    sb = new StringBuilder();
+    condition.appendMetricNameClause(sb);
+    Assert.assertEquals(sb.toString(), "(METRIC_NAME NOT LIKE ? AND METRIC_NAME NOT LIKE ?)");
+    Assert.assertTrue(CollectionUtils.isEqualCollection(metricNames, condition.getMetricNames()));
+
+    metricNames.clear();
+
+    // IN followed by LIKE clause
+    metricNames.add("M1");
+    metricNames.add("disk%");
+    metricNames.add("M2");
+    condition = new DefaultCondition(metricNames,null,null,null,null,null,null,null,true);
+    sb = new StringBuilder();
+    condition.appendMetricNameClause(sb);
+    Assert.assertEquals(sb.toString(), "(METRIC_NAME IN (?, ?) OR METRIC_NAME LIKE ?)");
+    Assert.assertEquals(metricNames.get(2), "disk%");
+
+    metricNames.clear();
+    //NOT IN followed by NOT LIKE clause
+    metricNames.add("disk%");
+    metricNames.add("metric1");
+    metricNames.add("cpu%");
+    condition = new DefaultCondition(metricNames,null,null,null,null,null,null,null,true);
+    sb = new StringBuilder();
+    condition.setMetricNamesNotCondition(true);
+    condition.appendMetricNameClause(sb);
+    Assert.assertEquals(sb.toString(), "(METRIC_NAME NOT IN (?) AND METRIC_NAME NOT LIKE ? AND METRIC_NAME NOT LIKE ?)");
+    Assert.assertEquals(metricNames.get(0), "metric1");
+
+    //Empty
+    metricNames.clear();
+    condition = new DefaultCondition(metricNames,null,null,null,null,null,null,null,true);
+    sb = new StringBuilder();
+    condition.appendMetricNameClause(sb);
+    Assert.assertEquals(sb.toString(), "");
+
+  }
+}
+

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml

@@ -660,4 +660,25 @@
     </description>
     <on-ambari-upgrade add="true"/>
   </property>
+  <property>
+    <name>timeline.metrics.service.metadata.filters</name>
+    <value>ContainerResource</value>
+    <description>
+      Commas separated list of regular expressions that match metric names
+      which prevents certain metrics from ending up in metadata cache.
+    </description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>timeline.metrics.cluster.aggregation.sql.filters</name>
+    <value>sdisk\_%,boottime</value>
+    <description>
+      Commas separated list of Metric names or Phoenix 'LIKE' class expressions that match metric names
+      which prevents certain metrics from being aggregated across hosts.
+    </description>
+    <on-ambari-upgrade add="true"/>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+  </property>
 </configuration>