AMBARI-18093 : Reduce TTL for high precision tables in AMS. (avijayan)

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java

@@ -196,6 +196,8 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
   private static final String AMS_HBASE_SITE = "ams-hbase-site";
   private static final String HBASE_RPC_TIMEOUT_PROPERTY = "hbase.rpc.timeout";
   private static final String AMS_HBASE_SITE_NORMALIZER_ENABLED_PROPERTY = "hbase.normalizer.enabled";
+  public static final String PRECISION_TABLE_TTL_PROPERTY = "timeline.metrics.host.aggregator.ttl";
+  public static final String CLUSTER_SECOND_TABLE_TTL_PROPERTY = "timeline.metrics.cluster.aggregator.second.ttl";
 
   static {
     // Manually create role order since there really isn't any mechanism for this
@@ -2008,6 +2010,19 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
               newProperties.put(TIMELINE_METRICS_SINK_COLLECTION_PERIOD, "10");
               LOG.info("Setting value of " + TIMELINE_METRICS_SINK_COLLECTION_PERIOD + " : 10");
             }
+
+            if (amsSiteProperties.containsKey(PRECISION_TABLE_TTL_PROPERTY) &&
+              !amsSiteProperties.get(PRECISION_TABLE_TTL_PROPERTY).equals("86400")) {
+              newProperties.put(PRECISION_TABLE_TTL_PROPERTY, String.valueOf(86400));
+              LOG.info("Setting value of " + PRECISION_TABLE_TTL_PROPERTY + " : " + 86400);
+            }
+
+            if (amsSiteProperties.containsKey(CLUSTER_SECOND_TABLE_TTL_PROPERTY) &&
+              !amsSiteProperties.get(CLUSTER_SECOND_TABLE_TTL_PROPERTY).equals("259200")) {
+              newProperties.put(CLUSTER_SECOND_TABLE_TTL_PROPERTY, String.valueOf(259200));
+              LOG.info("Setting value of " + CLUSTER_SECOND_TABLE_TTL_PROPERTY + " : " + 259200);
+            }
+
             updateConfigurationPropertiesForCluster(cluster, AMS_SITE, newProperties, true, true);
           }
 

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml

@@ -333,9 +333,9 @@
   </property>
   <property>
     <name>timeline.metrics.cluster.aggregator.second.ttl</name>
-    <value>604800</value>
+    <value>259200</value>
     <description>
-      Cluster wide second resolution data purge interval in seconds. Default is 7 days.
+      Cluster wide second resolution data purge interval in seconds. Default is 3 days.
     </description>
     <on-ambari-upgrade add="true"/>
   </property>
@@ -367,7 +367,7 @@
     <name>timeline.metrics.host.aggregator.ttl</name>
     <value>86400</value>
     <description>
-      1 minute resolution data purge interval in seconds. Default is 1 day for embedded mode and 3 days for Distributed mode.
+      1 minute resolution data purge interval in seconds. Default is 1 day.
     </description>
     <depends-on>
       <property>

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py

@@ -606,11 +606,9 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
 
     if operatingMode == "distributed":
       putAmsSiteProperty("timeline.metrics.service.watcher.disabled", 'true')
-      putAmsSiteProperty("timeline.metrics.host.aggregator.ttl", 259200)
       putAmsHbaseSiteProperty("hbase.cluster.distributed", 'true')
     else:
       putAmsSiteProperty("timeline.metrics.service.watcher.disabled", 'false')
-      putAmsSiteProperty("timeline.metrics.host.aggregator.ttl", 86400)
       putAmsHbaseSiteProperty("hbase.cluster.distributed", 'false')
 
     rootDir = "file:///var/lib/ambari-metrics-collector/hbase"

ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java

@@ -1299,11 +1299,15 @@ public class UpgradeCatalog240Test {
     Map<String, String> oldPropertiesAmsSite = new HashMap<String, String>() {
       {
         put("timeline.metrics.sink.collection.period", "60");
+        put("timeline.metrics.host.aggregator.ttl", "86400");
+        put("timeline.metrics.cluster.aggregator.second.ttl", "604800");
       }
     };
     Map<String, String> newPropertiesAmsSite = new HashMap<String, String>() {
       {
         put("timeline.metrics.sink.collection.period", "10");
+        put("timeline.metrics.host.aggregator.ttl", "86400");
+        put("timeline.metrics.cluster.aggregator.second.ttl", "259200");
       }
     };
     EasyMockSupport easyMockSupport = new EasyMockSupport();

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -2292,7 +2292,6 @@ class TestHDP22StackAdvisor(TestCase):
         "properties": {
           "timeline.metrics.cluster.aggregate.splitpoints": "master.FileSystem.MetaHlogSplitTime_75th_percentile",
           "timeline.metrics.host.aggregate.splitpoints": "master.FileSystem.MetaHlogSplitTime_75th_percentile",
-          "timeline.metrics.host.aggregator.ttl": "86400",
           "timeline.metrics.service.handler.thread.count": "20",
           'timeline.metrics.service.webapp.address': 'host1:6188',
           'timeline.metrics.service.watcher.disabled': 'false'
@@ -2477,7 +2476,6 @@ class TestHDP22StackAdvisor(TestCase):
     expected['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '512'
     expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '102'
     expected['ams-hbase-env']['properties']['regionserver_xmn_size'] = '384'
-    expected['ams-site']['properties']['timeline.metrics.host.aggregator.ttl'] = '259200'
     expected['ams-site']['properties']['timeline.metrics.service.watcher.disabled'] = 'true'
     self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)