AMBARI-3158. Ambari Hadoop: Job Summary Log should be configured to log as separate file(Dmytro Shkvyra via odiachenko)

ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/log4j.properties.erb

@@ -169,16 +169,25 @@ log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
 # hadoop.mapreduce.jobsummary.log.file rolled daily:
 # hadoop.mapreduce.jobsummary.logger=INFO,JSA
 # 
-hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
+hadoop.mapreduce.jobsummary.logger=<%=scope.function_hdp_default(["mapred-site/mapreduce.reduce.log.level","INFO"])%>,JSA
 hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
 log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file}
+<% if (scope.function_hdp_get_major_stack_version([scope.function_hdp_template_var("stack_version")]) >= 2)%>
+log4j.appender.JSA.File=<%=scope.function_hdp_template_var("yarn_log_dir_prefix")%>/<%=scope.function_hdp_template_var("yarn_user")%>/${hadoop.mapreduce.jobsummary.log.file}
+<% else %>
+log4j.appender.JSA.File=<%=scope.function_hdp_template_var("hdfs_log_dir_prefix")%>/<%=scope.function_hdp_template_var("mapred_user")%>/${hadoop.mapreduce.jobsummary.log.file}
+<%end-%>
 log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
 log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
 log4j.appender.JSA.DatePattern=.yyyy-MM-dd
+log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
+<% if (scope.function_hdp_get_major_stack_version([scope.function_hdp_template_var("stack_version")]) >= 2)%>
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${hadoop.mapreduce.jobsummary.logger}
+log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false
+<% else %>
 log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger}
 log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false
-
+<%end-%>
 
 <%=scope.function_hdp_template_var("rca_prefix")%>ambari.jobhistory.database=<%=scope.function_hdp_host("ambari_db_rca_url")%>
 <%=scope.function_hdp_template_var("rca_prefix")%>ambari.jobhistory.driver=<%=scope.function_hdp_host("ambari_db_rca_driver")%>

ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/service.pp

@@ -30,15 +30,17 @@ define hdp-yarn::service(
   $security_enabled = $hdp::params::security_enabled
   
   if ($name == 'historyserver') {
-    $log_dir = "${hdp-yarn::params::mapred_log_dir_prefix}"
+    $log_dir = "${hdp-yarn::params::mapred_log_dir_prefix}/${user}"
     $pid_dir = "${hdp-yarn::params::mapred_pid_dir_prefix}/${user}"
     $daemon = "${hdp::params::mapred_bin}/mr-jobhistory-daemon.sh"
     $pid_file = "${pid_dir}/mapred-${user}-${name}.pid"
+    $job_summary_log = "${hdp-yarn::params::mapred_log_dir_prefix}/${user}/hadoop-mapreduce.jobsummary.log"
   } else {
-    $log_dir = "${hdp-yarn::params::yarn_log_dir_prefix}"
+    $log_dir = "${hdp-yarn::params::yarn_log_dir_prefix}/${user}"
     $pid_dir = "${hdp-yarn::params::yarn_pid_dir_prefix}/${user}"
     $daemon = "${hdp::params::yarn_bin}/yarn-daemon.sh"
     $pid_file = "${pid_dir}/yarn-${user}-${name}.pid"
+    $job_summary_log = "${hdp-yarn::params::yarn_log_dir_prefix}/${user}/hadoop-mapreduce.jobsummary.log"
   }
   
   $hadoop_libexec_dir = $hdp-yarn::params::hadoop_libexec_dir
@@ -80,6 +82,11 @@ define hdp-yarn::service(
       service_state => $service_state,
       force => true
     }
+    
+    file {$job_summary_log:
+      path => $job_summary_log,
+      owner => $user,
+    }
   }
  
   if ($daemon_cmd != undef) {  
@@ -93,7 +100,7 @@ define hdp-yarn::service(
   anchor{"hdp-yarn::service::${name}::begin":}
   anchor{"hdp-yarn::service::${name}::end":}
   if ($daemon_cmd != undef) {
-    Anchor["hdp-yarn::service::${name}::begin"] -> Hdp::Directory_recursive_create<|title == $pid_dir or title == $log_dir|> -> Hdp::Exec[$daemon_cmd] -> Anchor["hdp-yarn::service::${name}::end"]
+    Anchor["hdp-yarn::service::${name}::begin"] -> Hdp::Directory_recursive_create<|title == $pid_dir or title == $log_dir|> -> File[$job_summary_log] -> Hdp::Exec[$daemon_cmd] -> Anchor["hdp-yarn::service::${name}::end"]
 
   }
   if ($ensure == 'running') {