YARN-6931. Make the aggregation interval in AppLevelTimelineCollector configurable. (Abhishek Modi via Haibo Chen)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -2664,6 +2664,15 @@ public class YarnConfiguration extends Configuration {
   public static final String TIMELINE_SERVICE_READ_AUTH_ENABLED =
       TIMELINE_SERVICE_PREFIX + "read.authentication.enabled";
 
+  /**
+   * The name for setting that controls how often in-memory app level
+   * aggregation is kicked off in timeline collector.
+   */
+  public static final String TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS =
+      TIMELINE_SERVICE_PREFIX + "app-aggregation-interval-secs";
+
+  public static final int
+      DEFAULT_TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS = 15;
   /**
    * The default setting for authentication checks for reading timeline
    * service v2 data.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -2545,6 +2545,15 @@
     <value>259200000</value>
   </property>
 
+  <property>
+    <description>
+      The setting that controls how often in-memory app level
+      aggregation is kicked off in timeline collector.
+    </description>
+    <name>yarn.timeline-service.app-aggregation-interval-secs</name>
+    <value>15</value>
+  </property>
+
   <property>
     <description>
     The default hdfs location for flowrun coprocessor jar.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollectorWithAgg.java

@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -50,7 +51,7 @@ public class AppLevelTimelineCollectorWithAgg
       LoggerFactory.getLogger(TimelineCollector.class);
 
   private final static int AGGREGATION_EXECUTOR_NUM_THREADS = 1;
-  private final static int AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS = 15;
+  private int aggregationExecutorIntervalSecs;
   private static Set<String> entityTypesSkipAggregation
       = initializeSkipSet();
 
@@ -71,6 +72,11 @@ public class AppLevelTimelineCollectorWithAgg
 
   @Override
   protected void serviceInit(Configuration conf) throws Exception {
+    aggregationExecutorIntervalSecs = conf.getInt(
+        YarnConfiguration.TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS,
+        YarnConfiguration.
+            DEFAULT_TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS
+    );
     super.serviceInit(conf);
   }
 
@@ -84,10 +90,8 @@ public class AppLevelTimelineCollectorWithAgg
             .build());
     appAggregator = new AppLevelAggregator();
     appAggregationExecutor.scheduleAtFixedRate(appAggregator,
-        AppLevelTimelineCollectorWithAgg.
-          AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS,
-        AppLevelTimelineCollectorWithAgg.
-          AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS,
+        aggregationExecutorIntervalSecs,
+        aggregationExecutorIntervalSecs,
         TimeUnit.SECONDS);
     super.serviceStart();
   }