AMBARI-15892 : Incorrect (Negative) values are shown for memory metrics - Commit 2 (avijayan)

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java

@@ -72,6 +72,7 @@ import java.util.concurrent.TimeUnit;
 
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.AGGREGATE_TABLE_SPLIT_POINTS;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HBASE_BLOCKING_STORE_FILES;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_TABLES_DURABILITY;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.AGGREGATORS_SKIP_BLOCK_CACHE;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_DAILY_TABLE_TTL;
@@ -405,9 +406,13 @@ public class PhoenixHBaseAccessor {
             LOG.info("Setting config property " + HSTORE_COMPACTION_CLASS_KEY +
               " = " + FIFO_COMPACTION_POLICY_CLASS + " for " + tableName);
             // Need to set blockingStoreFiles to 1000 for FIFO
-            tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, "1000");
+            int blockingStoreFiles = hbaseConf.getInt(HBASE_BLOCKING_STORE_FILES, 1000);
+            if (blockingStoreFiles < 1000) {
+              blockingStoreFiles = 1000;
+            }
+            tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, String.valueOf(blockingStoreFiles));
             LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY +
-              " = " + 1000 + " for " + tableName);
+              " = " + blockingStoreFiles + " for " + tableName);
             modifyTable = true;
           }
           // Set back original policy if fifo disabled
@@ -417,9 +422,16 @@ public class PhoenixHBaseAccessor {
               DEFAULT_COMPACTION_POLICY_CLASS);
             LOG.info("Setting config property " + HSTORE_COMPACTION_CLASS_KEY +
               " = " + DEFAULT_COMPACTION_POLICY_CLASS + " for " + tableName);
-            tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, "300");
+
+            int blockingStoreFiles = hbaseConf.getInt(HBASE_BLOCKING_STORE_FILES, 300);
+            if (blockingStoreFiles > 300) {
+              LOG.warn("HBase blocking store file set too high without FIFO " +
+                "Compaction policy enabled, restoring low value = 300.");
+              blockingStoreFiles = 300;
+            }
+            tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, String.valueOf(blockingStoreFiles));
             LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY +
-              " = " + 300 + " for " + tableName);
+              " = " + blockingStoreFiles + " for " + tableName);
             modifyTable = true;
           }
           // Change TTL setting to match user configuration

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java

@@ -231,6 +231,9 @@ public class TimelineMetricConfiguration {
   public static final String TIMELINE_METRIC_METADATA_FILTERS =
     "timeline.metrics.service.metadata.filters";
 
+  public static final String HBASE_BLOCKING_STORE_FILES =
+    "hbase.hstore.blockingStoreFiles";
+
   public static final String HOST_APP_ID = "HOST";
 
   public static final String DEFAULT_INSTANCE_PORT = "12001";

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java

@@ -107,6 +107,7 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
   public static final String PHOENIX_QUERY_KEEPALIVE_PROPERTY = "phoenix.query.keepAliveMs";
   public static final String TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED
     = "timeline.metrics.cluster.aggregator.interpolation.enabled";
+  public static final String TIMELINE_METRICS_SINK_COLLECTION_PERIOD = "timeline.metrics.sink.collection.period";
 
   public static final String AMS_SERVICE_NAME = "AMBARI_METRICS";
   public static final String AMS_COLLECTOR_COMPONENT_NAME = "METRICS_COLLECTOR";
@@ -399,6 +400,13 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
               newProperties.put(TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED, String.valueOf(true));
             }
 
+            if (!amsSiteProperties.containsKey(TIMELINE_METRICS_SINK_COLLECTION_PERIOD) ||
+              "60".equals(amsSiteProperties.get(TIMELINE_METRICS_SINK_COLLECTION_PERIOD))) {
+
+              newProperties.put(TIMELINE_METRICS_SINK_COLLECTION_PERIOD, "10");
+              LOG.info("Setting value of " + TIMELINE_METRICS_SINK_COLLECTION_PERIOD + " : 10");
+            }
+
             updateConfigurationPropertiesForCluster(cluster, AMS_SITE, newProperties, true, true);
           }
 
@@ -457,7 +465,7 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
     Map<String, List<String>> widgetMap = new HashMap<>();
     Map<String, String> sectionLayoutMap = new HashMap<>();
 
-    List<String> yarnSummaryWidgets = new ArrayList<>(Arrays.asList("Container Failures", "App Failures"));
+    List<String> yarnSummaryWidgets = new ArrayList<>(Arrays.asList("Container Failures", "App Failures", "Cluster Memory"));
     widgetMap.put("YARN_SUMMARY", yarnSummaryWidgets);
     sectionLayoutMap.put("YARN_SUMMARY", "default_yarn_dashboard");
 
@@ -582,6 +590,10 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
               if (targetWidgetLayoutInfo != null) {
                 entityToUpdate.setMetrics(gson.toJson(targetWidgetLayoutInfo.getMetricsInfo()));
                 entityToUpdate.setWidgetValues(gson.toJson(targetWidgetLayoutInfo.getValues()));
+                if ("HBASE".equals(serviceName) && "Reads and Writes".equals(widgetName)) {
+                  entityToUpdate.setDescription(targetWidgetLayoutInfo.getDescription());
+                  LOG.info("Update description for HBase Reads and Writes widget");
+                }
                 widgetDAO.merge(entityToUpdate);
               } else {
                 LOG.warn("Unable to find widget layout info for " + widgetName +

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/widgets.json

@@ -7,7 +7,7 @@
       "widgetLayoutInfo": [
         {
           "widget_name": "Reads and Writes",
-          "description": "Count of read and write requests on all regions in the cluster.",
+          "description": "Rate (per second) of read and write requests on all regions in the cluster.",
           "default_section_name": "HBASE_SUMMARY",
           "widget_type": "GRAPH",
           "is_visible": true,

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/YARN_widgets.json

@@ -230,18 +230,12 @@
               "metric_path": "metrics/memory/mem_free._avg",
               "service_name": "YARN",
               "component_name": "NODEMANAGER"
-            },
-            {
-              "name": "mem_cached._sum",
-              "metric_path": "metrics/memory/mem_cached._avg",
-              "service_name": "YARN",
-              "component_name": "NODEMANAGER"
             }
           ],
           "values": [
             {
               "name": "Memory utilization",
-              "value": "${((mem_total._sum - mem_free._sum - mem_cached._sum)/mem_total._sum) * 100}"
+              "value": "${((mem_total._sum - mem_free._sum)/mem_total._sum) * 100}"
             }
           ],
           "properties": {

ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/widgets.json

@@ -7,7 +7,7 @@
       "widgetLayoutInfo": [
         {
           "widget_name": "Reads and Writes",
-          "description": "Count of read and write requests on all regions in the cluster.",
+          "description": "Rate (per second) of read and write requests on all regions in the cluster.",
           "widget_type": "GRAPH",
           "is_visible": true,
           "metrics": [

ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/YARN_widgets.json

@@ -259,18 +259,12 @@
               "metric_path": "metrics/memory/mem_free._avg",
               "service_name": "YARN",
               "component_name": "NODEMANAGER"
-            },
-            {
-              "name": "mem_cached._sum",
-              "metric_path": "metrics/memory/mem_cached._avg",
-              "service_name": "YARN",
-              "component_name": "NODEMANAGER"
             }
           ],
           "values": [
             {
               "name": "Memory utilization",
-              "value": "${((mem_total._sum - mem_free._sum - mem_cached._sum)/mem_total._sum) * 100}"
+              "value": "${((mem_total._sum - mem_free._sum)/mem_total._sum) * 100}"
             }
           ],
           "properties": {

ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java

@@ -349,14 +349,10 @@ public class UpgradeCatalog222Test {
         put("timeline.metrics.cluster.aggregator.daily.checkpointCutOffMultiplier", String.valueOf(1));
         put("timeline.metrics.service.operation.mode", "distributed");
         put("timeline.metrics.host.aggregator.ttl", String.valueOf(86400));
-        put("timeline.metrics.host.aggregator.minute.ttl", String.valueOf(604800));
-        put("timeline.metrics.host.aggregator.hourly.ttl", String.valueOf(2592000));
-        put("timeline.metrics.host.aggregator.daily.ttl", String.valueOf(31536000));
         put("timeline.metrics.cluster.aggregator.second.ttl", String.valueOf(21600)); //Less than 1 day
         put("timeline.metrics.cluster.aggregator.minute.ttl", String.valueOf(7776000));
-        put("timeline.metrics.cluster.aggregator.hourly.ttl", String.valueOf(31536000));
-        put("timeline.metrics.cluster.aggregator.daily.ttl", String.valueOf(63072000));
         put("timeline.metrics.service.webapp.address", "0.0.0.0:6188");
+        put("timeline.metrics.sink.collection.period", "60");
       }
     };
     Map<String, String> newPropertiesAmsSite = new HashMap<String, String>() {
@@ -365,16 +361,12 @@ public class UpgradeCatalog222Test {
         put("timeline.metrics.cluster.aggregator.daily.checkpointCutOffMultiplier", String.valueOf(2));
         put("timeline.metrics.service.watcher.disabled", String.valueOf(false));
         put("timeline.metrics.host.aggregator.ttl", String.valueOf(3 * 86400));
-        put("timeline.metrics.host.aggregator.minute.ttl", String.valueOf(7 * 86400));
-        put("timeline.metrics.host.aggregator.hourly.ttl", String.valueOf(30 * 86400));
-        put("timeline.metrics.host.aggregator.daily.ttl", String.valueOf(365 * 86400));
         put("timeline.metrics.cluster.aggregator.second.ttl", String.valueOf(21600));
         put("timeline.metrics.cluster.aggregator.minute.ttl", String.valueOf(30 * 86400));
-        put("timeline.metrics.cluster.aggregator.hourly.ttl", String.valueOf(365 * 86400));
-        put("timeline.metrics.cluster.aggregator.daily.ttl", String.valueOf(730 * 86400));
         put("timeline.metrics.service.operation.mode", "distributed");
         put("timeline.metrics.service.webapp.address", "host1:6188");
         put("timeline.metrics.cluster.aggregator.interpolation.enabled", String.valueOf(true));
+        put("timeline.metrics.sink.collection.period", "10");
       }
     };
     EasyMockSupport easyMockSupport = new EasyMockSupport();