AMBARI-3537 Allow log4j properties to be applied via the API in Ambari for hadoop/oozie/hbase/hive/zookeeper/pig (dsen)

ambari-agent/src/main/python/resource_management/libraries/providers/__init__.py

@@ -29,6 +29,7 @@ PROVIDERS = dict(
     ExecuteHadoop="resource_management.libraries.providers.execute_hadoop.ExecuteHadoopProvider",
     TemplateConfig="resource_management.libraries.providers.template_config.TemplateConfigProvider",
     XmlConfig="resource_management.libraries.providers.xml_config.XmlConfigProvider",
+    PropertiesFile="resource_management.libraries.providers.properties_file.PropertiesFileProvider",
     MonitorWebserver="resource_management.libraries.providers.monitor_webserver.MonitorWebserverProvider",
     Repository="resource_management.libraries.providers.repository.RepositoryProvider"
   ),

ambari-agent/src/main/python/resource_management/libraries/providers/properties_file.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import time
+import os
+from resource_management import *
+
+class PropertiesFileProvider(Provider):
+  def action_create(self):
+    filename = self.resource.filename
+    dir = self.resource.dir
+    if dir == None:
+      filepath = filename
+    else:
+      filepath = os.path.join(dir, filename)
+
+    config_content = InlineTemplate('''# Generated by Apache Ambari. {{time.asctime(time.localtime())}}
+    {% for key, value in properties_dict|dictsort %}
+{{key}}={{value}}{% endfor %}
+    ''', extra_imports=[time], properties_dict=self.resource.properties)
+
+    self.log.debug(format("Generating properties file: {filepath}"))
+
+    with Environment.get_instance_copy() as env:
+      File (format("{filepath}"),
+            content = config_content,
+            owner = self.resource.owner,
+            group = self.resource.group,
+            mode = self.resource.mode
+      )

ambari-agent/src/main/python/resource_management/libraries/resources/__init__.py

@@ -23,5 +23,6 @@ Ambari Agent
 from resource_management.libraries.resources.execute_hadoop import *
 from resource_management.libraries.resources.template_config import *
 from resource_management.libraries.resources.xml_config import *
+from resource_management.libraries.resources.properties_file import *
 from resource_management.libraries.resources.repository import *
 from resource_management.libraries.resources.monitor_webserver import *

ambari-agent/src/main/python/resource_management/libraries/resources/properties_file.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+_all__ = ["PropertiesFile"]
+from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument
+
+class PropertiesFile(Resource):
+  action = ForcedListArgument(default="create")
+  filename = ResourceArgument(default=lambda obj: obj.name)
+
+  properties = ResourceArgument()
+  dir = ResourceArgument()
+
+  mode = ResourceArgument()
+  owner = ResourceArgument()
+  group = ResourceArgument()
+
+  actions = Resource.actions + ["create"]

ambari-server/src/main/resources/stacks/HDP/1.3.3/hooks/before-START/scripts/params.py

@@ -169,4 +169,29 @@ if dfs_ha_enabled:
       namenode_id = nn_id
   namenode_id = None
 
-dfs_hosts = default('/configurations/hdfs-site/dfs.hosts', None)
+dfs_hosts = default('/configurations/hdfs-site/dfs.hosts', None)
+
+#log4j.properties
+rca_property_map = {
+  'ambari.jobhistory.database': ambari_db_rca_url,
+  'ambari.jobhistory.driver': ambari_db_rca_driver,
+  'ambari.jobhistory.user': ambari_db_rca_username,
+  'ambari.jobhistory.password': ambari_db_rca_password,
+  'ambari.jobhistory.logger': 'DEBUG,JHA',
+
+  'log4j.appender.JHA': 'org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender',
+  'log4j.appender.JHA.database': '${ambari.jobhistory.database}',
+  'log4j.appender.JHA.driver': '${ambari.jobhistory.driver}',
+  'log4j.appender.JHA.user': '${ambari.jobhistory.user}',
+  'log4j.appender.JHA.password': '${ambari.jobhistory.password}',
+
+  'log4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger': '${ambari.jobhistory.logger}',
+  'log4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger': 'true'
+}
+
+if ('hdfs-log4j' in config['configurations']):
+  log4j_props = config['configurations']['hdfs-log4j']
+  if 'mapreduce-log4j' in config['configurations']:
+    log4j_props.update(config['configurations']['mapreduce-log4j'])
+else:
+  log4j_props = None

ambari-server/src/main/resources/stacks/HDP/1.3.3/hooks/before-START/scripts/shared_initialization.py

@@ -141,12 +141,22 @@ def setup_hadoop():
        content=Template(health_check_template + ".j2")
   )
 
-  File(os.path.join(params.hadoop_conf_dir, "log4j.properties"),
-       owner=params.hdfs_user,
-       content=Template("log4j.properties.j2")
-  )
+  log4j_filename = os.path.join(params.hadoop_conf_dir, "log4j.properties")
+  if (params.log4j_props != None):
+    PropertiesFile(log4j_filename,
+                   properties=params.log4j_props,
+                   mode=0664,
+                   owner=params.hdfs_user,
+                   group=params.user_group,
+    )
+  elif (os.path.exists(format("{params.hadoop_conf_dir}/log4j.properties"))):
+    File(log4j_filename,
+         mode=0644,
+         group=params.user_group,
+         owner=params.hdfs_user,
+    )
 
-  update_log4j_props(os.path.join(params.hadoop_conf_dir, "log4j.properties"))
+  update_log4j_props(log4j_filename)
 
   File(os.path.join(params.hadoop_conf_dir, "hadoop-metrics2.properties"),
        owner=params.hdfs_user,
@@ -270,26 +280,11 @@ def setup_configs():
 def update_log4j_props(file):
   import params
 
-  property_map = {
-    'ambari.jobhistory.database': params.ambari_db_rca_url,
-    'ambari.jobhistory.driver': params.ambari_db_rca_driver,
-    'ambari.jobhistory.user': params.ambari_db_rca_username,
-    'ambari.jobhistory.password': params.ambari_db_rca_password,
-    'ambari.jobhistory.logger': 'DEBUG,JHA',
-
-    'log4j.appender.JHA': 'org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender',
-    'log4j.appender.JHA.database': '${ambari.jobhistory.database}',
-    'log4j.appender.JHA.driver': '${ambari.jobhistory.driver}',
-    'log4j.appender.JHA.user': '${ambari.jobhistory.user}',
-    'log4j.appender.JHA.password': '${ambari.jobhistory.password}',
-
-    'log4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger': '${ambari.jobhistory.logger}',
-    'log4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger': 'true'
-  }
-  for key in property_map:
-    value = property_map[key]
+  for key in params.rca_property_map:
+    value = params.rca_property_map[key]
     Execute(format(
-      "sed -i 's~\\({rca_disabled_prefix}\\)\\?{key}=.*~{rca_prefix}{key}={value}~' {file}"))
+      "sed -i 's~\\({rca_disabled_prefix}\\)\\?{key}=.*~{rca_prefix}{key}={value}~' {file}")
+    )
 
 
 def generate_include_file():

ambari-server/src/main/resources/stacks/HDP/1.3.3/hooks/before-START/templates/log4j.properties.j2

@@ -1,200 +0,0 @@
-# Copyright 2011 The Apache Software Foundation
-# 
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Define some default values that can be overridden by system properties
-hadoop.root.logger=INFO,console
-hadoop.log.dir=.
-hadoop.log.file=hadoop.log
-
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hadoop.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshhold=ALL
-
-#
-# Daily Rolling File Appender
-#
-
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
-
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-# 30-day backup
-#log4j.appender.DRFA.MaxBackupIndex=30
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this 
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-
-#
-# TaskLog Appender
-#
-
-#Default values
-hadoop.tasklog.taskid=null
-hadoop.tasklog.iscleanup=false
-hadoop.tasklog.noKeepSplits=4
-hadoop.tasklog.totalLogFileSize=100
-hadoop.tasklog.purgeLogSplits=true
-hadoop.tasklog.logsRetainHours=12
-
-log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
-log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
-log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}
-log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
-
-log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
-log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-
-#
-#Security audit appender
-#
-hadoop.security.logger=INFO,console
-hadoop.security.log.maxfilesize=256MB
-hadoop.security.log.maxbackupindex=20
-log4j.category.SecurityLogger=${hadoop.security.logger}
-hadoop.security.log.file=SecurityAuth.audit
-log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender 
-log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
-log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
-log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd
-
-log4j.appender.RFAS=org.apache.log4j.RollingFileAppender 
-log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
-log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}
-log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}
-
-#
-# hdfs audit logging
-#
-hdfs.audit.logger=INFO,console
-log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}
-log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false
-log4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log
-log4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd
-
-#
-# mapred audit logging
-#
-mapred.audit.logger=INFO,console
-log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}
-log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false
-log4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log
-log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd
-
-#
-# Rolling File Appender
-#
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
-
-# Logfile size and and 30-day backups
-log4j.appender.RFA.MaxFileSize=256MB
-log4j.appender.RFA.MaxBackupIndex=10
-
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-# Custom Logging levels
-
-hadoop.metrics.log.level=INFO
-#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
-#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
-#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
-log4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}
-
-# Jets3t library
-log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
-
-#
-# Null Appender
-# Trap security logger on the hadoop client side
-#
-log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
-
-{% if is_jtnode_master or is_rmnode_master %}
-#
-# Job Summary Appender 
-#
-# Use following logger to send summary to separate file defined by 
-# hadoop.mapreduce.jobsummary.log.file rolled daily:
-# hadoop.mapreduce.jobsummary.logger=INFO,JSA
-# 
-hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
-hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
-log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender
-
-log4j.appender.JSA.File={{hdfs_log_dir_prefix}}/{{mapred_user}}/${hadoop.mapreduce.jobsummary.log.file}
-
-log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
-log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-log4j.appender.JSA.DatePattern=.yyyy-MM-dd
-log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
-log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger}
-log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false
-{% endif %}
-
-{{rca_prefix}}ambari.jobhistory.database={{ambari_db_rca_url}}
-{{rca_prefix}}ambari.jobhistory.driver={{ambari_db_rca_driver}}
-{{rca_prefix}}ambari.jobhistory.user={{ambari_db_rca_username}}
-{{rca_prefix}}ambari.jobhistory.password={{ambari_db_rca_password}}
-{{rca_prefix}}ambari.jobhistory.logger=DEBUG,JHA
-
-{{rca_prefix}}log4j.appender.JHA=org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender
-{{rca_prefix}}log4j.appender.JHA.database=${ambari.jobhistory.database}
-{{rca_prefix}}log4j.appender.JHA.driver=${ambari.jobhistory.driver}
-{{rca_prefix}}log4j.appender.JHA.user=${ambari.jobhistory.user}
-{{rca_prefix}}log4j.appender.JHA.password=${ambari.jobhistory.password}
-
-{{rca_prefix}}log4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=${ambari.jobhistory.logger}
-{{rca_prefix}}log4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=true

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/HBASE/configuration/hbase-log4j.xml

@@ -0,0 +1,183 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+
+  <property>
+    <name>hbase.root.logger</name>
+    <value>INFO,console</value>
+  </property>
+  <property>
+    <name>hbase.security.logger</name>
+    <value>INFO,console</value>
+  </property>
+  <property>
+    <name>hbase.log.dir</name>
+    <value>.</value>
+  </property>
+  <property>
+    <name>hbase.log.file</name>
+    <value>hbase.log</value>
+  </property>
+  <property>
+    <name>log4j.rootLogger</name>
+    <value>${hbase.root.logger}</value>
+  </property>
+  <property>
+    <name>log4j.threshold</name>
+    <value>ALL</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.File</name>
+    <value>${hbase.log.dir}/${hbase.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %-5p [%t] %c{2}: %m%n</value>
+  </property>
+  <property>
+    <name>hbase.log.maxfilesize</name>
+    <value>256MB</value>
+  </property>
+  <property>
+    <name>hbase.log.maxbackupindex</name>
+    <value>20</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA</name>
+    <value>org.apache.log4j.RollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.File</name>
+    <value>${hbase.log.dir}/${hbase.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.MaxFileSize</name>
+    <value>${hbase.log.maxfilesize}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.MaxBackupIndex</name>
+    <value>${hbase.log.maxbackupindex}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %-5p [%t] %c{2}: %m%n</value>
+  </property>
+  <property>
+    <name>hbase.security.log.file</name>
+    <value>SecurityAuth.audit</value>
+  </property>
+  <property>
+    <name>hbase.security.log.maxfilesize</name>
+    <value>256MB</value>
+  </property>
+  <property>
+    <name>hbase.security.log.maxbackupindex</name>
+    <value>20</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS</name>
+    <value>org.apache.log4j.RollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.File</name>
+    <value>${hbase.log.dir}/${hbase.security.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.MaxFileSize</name>
+    <value>${hbase.security.log.maxfilesize}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.MaxBackupIndex</name>
+    <value>${hbase.security.log.maxbackupindex}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %p %c: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.category.SecurityLogger</name>
+    <value>${hbase.security.logger}</value>
+  </property>
+  <property>
+    <name>log4j.additivity.SecurityLogger</name>
+    <value>false</value>
+  </property>
+  <property>
+    <name>log4j.appender.NullAppender</name>
+    <value>org.apache.log4j.varia.NullAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.console</name>
+    <value>org.apache.log4j.ConsoleAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.target</name>
+    <value>System.err</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %-5p [%t] %c{2}: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.zookeeper</name>
+    <value>INFO</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.hadoop.hbase</name>
+    <value>DEBUG</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil</name>
+    <value>INFO</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher</name>
+    <value>INFO</value>
+  </property>
+
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/HBASE/metainfo.xml

@@ -116,6 +116,7 @@
         <config-type>global</config-type>
         <config-type>hbase-policy</config-type>
         <config-type>hbase-site</config-type>
+        <config-type>hbase-log4j</config-type>
       </configuration-dependencies>
 
     </service>

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/HBASE/package/scripts/hbase.py

@@ -17,6 +17,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 
 """
+import os
 
 from resource_management import *
 import sys
@@ -78,7 +79,22 @@ def hbase(type=None # 'master' or 'regionserver' or 'client'
     Directory ( [params.tmp_dir, params.log_dir],
       owner = params.hbase_user,
       recursive = True
-    )    
+    )
+
+  if (params.log4j_props != None):
+    PropertiesFile('log4j.properties',
+                   dir=params.conf_dir,
+                   properties=params.log4j_props,
+                   mode=0664,
+                   owner=params.hbase_user,
+                   group=params.user_group,
+    )
+  elif (os.path.exists(format("{params.conf_dir}/log4j.properties"))):
+    File(format("{params.conf_dir}/log4j.properties"),
+         mode=0644,
+         group=params.user_group,
+         owner=params.hbase_user
+    )
 
 def hbase_TemplateConfig(name, 
                          tag=None

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/HBASE/package/scripts/params.py

@@ -83,3 +83,9 @@ regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regions
 smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
 hbase_user_keytab = config['configurations']['global']['hbase_user_keytab']
 kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+
+#log4j.properties
+if ('hbase-log4j' in config['configurations']):
+  log4j_props = config['configurations']['hbase-log4j']
+else:
+  log4j_props = None

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/HDFS/configuration/hdfs-log4j.xml

@@ -0,0 +1,305 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+  <property>
+    <name>hadoop.root.logger</name>
+    <value>INFO,console</value>
+  </property>
+  <property>
+    <name>hadoop.log.dir</name>
+    <value>.</value>
+  </property>
+  <property>
+    <name>hadoop.log.file</name>
+    <value>hadoop.log</value>
+  </property>
+  <property>
+    <name>log4j.rootLogger</name>
+    <value>${hadoop.root.logger}, EventCounter</value>
+  </property>
+  <property>
+    <name>log4j.threshhold</name>
+    <value>ALL</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.File</name>
+    <value>${hadoop.log.dir}/${hadoop.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %p %c: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.console</name>
+    <value>org.apache.log4j.ConsoleAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.target</name>
+    <value>System.err</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.layout.ConversionPattern</name>
+    <value>%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n</value>
+  </property>
+  <property>
+    <name>hadoop.tasklog.taskid</name>
+    <value>null</value>
+  </property>
+  <property>
+    <name>hadoop.tasklog.iscleanup</name>
+    <value>false</value>
+  </property>
+  <property>
+    <name>hadoop.tasklog.noKeepSplits</name>
+    <value>4</value>
+  </property>
+  <property>
+    <name>hadoop.tasklog.totalLogFileSize</name>
+    <value>100</value>
+  </property>
+  <property>
+    <name>hadoop.tasklog.purgeLogSplits</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>hadoop.tasklog.logsRetainHours</name>
+    <value>12</value>
+  </property>
+  <property>
+    <name>log4j.appender.TLA</name>
+    <value>org.apache.hadoop.mapred.TaskLogAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.TLA.taskId</name>
+    <value>${hadoop.tasklog.taskid}</value>
+  </property>
+  <property>
+    <name>log4j.appender.TLA.isCleanup</name>
+    <value>${hadoop.tasklog.iscleanup}</value>
+  </property>
+  <property>
+    <name>log4j.appender.TLA.totalLogFileSize</name>
+    <value>${hadoop.tasklog.totalLogFileSize}</value>
+  </property>
+  <property>
+    <name>log4j.appender.TLA.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.TLA.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %p %c: %m%n</value>
+  </property>
+  <property>
+    <name>hadoop.security.logger</name>
+    <value>INFO,console</value>
+  </property>
+  <property>
+    <name>hadoop.security.log.maxfilesize</name>
+    <value>256MB</value>
+  </property>
+  <property>
+    <name>hadoop.security.log.maxbackupindex</name>
+    <value>20</value>
+  </property>
+  <property>
+    <name>log4j.category.SecurityLogger</name>
+    <value>${hadoop.security.logger}</value>
+  </property>
+  <property>
+    <name>hadoop.security.log.file</name>
+    <value>SecurityAuth.audit</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAS</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAS.File</name>
+    <value>${hadoop.log.dir}/${hadoop.security.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAS.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAS.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %p %c: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAS.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS</name>
+    <value>org.apache.log4j.RollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.File</name>
+    <value>${hadoop.log.dir}/${hadoop.security.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %p %c: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.MaxFileSize</name>
+    <value>${hadoop.security.log.maxfilesize}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.MaxBackupIndex</name>
+    <value>${hadoop.security.log.maxbackupindex}</value>
+  </property>
+  <property>
+    <name>hdfs.audit.logger</name>
+    <value>INFO,console</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit</name>
+    <value>${hdfs.audit.logger}</value>
+  </property>
+  <property>
+    <name>log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit</name>
+    <value>false</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAAUDIT</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAAUDIT.File</name>
+    <value>${hadoop.log.dir}/hdfs-audit.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAAUDIT.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAAUDIT.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %p %c{2}: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAAUDIT.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>mapred.audit.logger</name>
+    <value>INFO,console</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.hadoop.mapred.AuditLogger</name>
+    <value>${mapred.audit.logger}</value>
+  </property>
+  <property>
+    <name>log4j.additivity.org.apache.hadoop.mapred.AuditLogger</name>
+    <value>false</value>
+  </property>
+  <property>
+    <name>log4j.appender.MRAUDIT</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.MRAUDIT.File</name>
+    <value>${hadoop.log.dir}/mapred-audit.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.MRAUDIT.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.MRAUDIT.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %p %c{2}: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.MRAUDIT.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA</name>
+    <value>org.apache.log4j.RollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.File</name>
+    <value>${hadoop.log.dir}/${hadoop.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.MaxFileSize</name>
+    <value>256MB</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.MaxBackupIndex</name>
+    <value>10</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %-5p %c{2} - %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n</value>
+  </property>
+  <property>
+    <name>hadoop.metrics.log.level</name>
+    <value>INFO</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.hadoop.metrics2</name>
+    <value>${hadoop.metrics.log.level}</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service</name>
+    <value>ERROR</value>
+  </property>
+  <property>
+    <name>log4j.appender.NullAppender</name>
+    <value>org.apache.log4j.varia.NullAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.EventCounter</name>
+    <value>org.apache.hadoop.log.metrics.EventCounter</value>
+  </property>
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/HDFS/metainfo.xml

@@ -140,6 +140,7 @@
         <config-type>global</config-type>
         <config-type>hdfs-site</config-type>
         <config-type>hadoop-policy</config-type>
+        <config-type>hdfs-log4j</config-type>
       </configuration-dependencies>
     </service>
   </services>

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/HIVE/configuration/hive-exec-log4j.xml

@@ -0,0 +1,122 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+
+  <property>
+    <name>hive.log.threshold</name>
+    <value>ALL</value>
+  </property>
+  <property>
+    <name>hive.root.logger</name>
+    <value>INFO,FA</value>
+  </property>
+  <property>
+    <name>hive.log.dir</name>
+    <value>/tmp/${user.name}</value>
+  </property>
+  <property>
+    <name>hive.log.file</name>
+    <value>${hive.query.id}.log</value>
+  </property>
+  <property>
+    <name>log4j.rootLogger</name>
+    <value>${hive.root.logger}, EventCounter</value>
+  </property>
+  <property>
+    <name>log4j.threshhold</name>
+    <value>${hive.log.threshold}</value>
+  </property>
+  <property>
+    <name>log4j.appender.FA</name>
+    <value>org.apache.log4j.FileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.FA.File</name>
+    <value>${hive.log.dir}/${hive.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.FA.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.FA.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.console</name>
+    <value>org.apache.log4j.ConsoleAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.target</name>
+    <value>System.err</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.layout.ConversionPattern</name>
+    <value>%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.EventCounter</name>
+    <value>org.apache.hadoop.hive.shims.HiveEventCounter</value>
+  </property>
+  <property>
+    <name>log4j.category.DataNucleus</name>
+    <value>ERROR,FA</value>
+  </property>
+  <property>
+    <name>log4j.category.Datastore</name>
+    <value>ERROR,FA</value>
+  </property>
+  <property>
+    <name>log4j.category.Datastore.Schema</name>
+    <value>ERROR,FA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.Datastore</name>
+    <value>ERROR,FA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.Plugin</name>
+    <value>ERROR,FA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.MetaData</name>
+    <value>ERROR,FA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.Query</name>
+    <value>ERROR,FA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.General</name>
+    <value>ERROR,FA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.Enhancer</name>
+    <value>ERROR,FA</value>
+  </property>
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/HIVE/configuration/hive-log4j.xml

@@ -0,0 +1,130 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+
+  <property>
+    <name>hive.log.threshold</name>
+    <value>ALL</value>
+  </property>
+  <property>
+    <name>hive.root.logger</name>
+    <value>WARN,DRFA</value>
+  </property>
+  <property>
+    <name>hive.log.dir</name>
+    <value>/tmp/${user.name}</value>
+  </property>
+  <property>
+    <name>hive.log.file</name>
+    <value>hive.log</value>
+  </property>
+  <property>
+    <name>log4j.rootLogger</name>
+    <value>${hive.root.logger}, EventCounter</value>
+  </property>
+  <property>
+    <name>log4j.threshold</name>
+    <value>${hive.log.threshold}</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.File</name>
+    <value>${hive.log.dir}/${hive.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.console</name>
+    <value>org.apache.log4j.ConsoleAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.target</name>
+    <value>System.err</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.layout.ConversionPattern</name>
+    <value>%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.encoding</name>
+    <value>UTF-8</value>
+  </property>
+  <property>
+    <name>log4j.appender.EventCounter</name>
+    <value>org.apache.hadoop.hive.shims.HiveEventCounter</value>
+  </property>
+  <property>
+    <name>log4j.category.DataNucleus</name>
+    <value>ERROR,DRFA</value>
+  </property>
+  <property>
+    <name>log4j.category.Datastore</name>
+    <value>ERROR,DRFA</value>
+  </property>
+  <property>
+    <name>log4j.category.Datastore.Schema</name>
+    <value>ERROR,DRFA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.Datastore</name>
+    <value>ERROR,DRFA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.Plugin</name>
+    <value>ERROR,DRFA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.MetaData</name>
+    <value>ERROR,DRFA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.Query</name>
+    <value>ERROR,DRFA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.General</name>
+    <value>ERROR,DRFA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.Enhancer</name>
+    <value>ERROR,DRFA</value>
+  </property>
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/HIVE/metainfo.xml

@@ -142,6 +142,8 @@
       <configuration-dependencies>
         <config-type>hive-site</config-type>
         <config-type>global</config-type>
+        <config-type>hive-log4j</config-type>
+        <config-type>hive-exec-log4j</config-type>
       </configuration-dependencies>
     </service>
 

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/HIVE/package/scripts/hive.py

@@ -17,6 +17,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 
 """
+import os
 
 from resource_management import *
 import sys
@@ -78,8 +79,40 @@ def hive(name=None):
 
   crt_file(format("{hive_conf_dir}/hive-default.xml.template"))
   crt_file(format("{hive_conf_dir}/hive-env.sh.template"))
-  crt_file(format("{hive_conf_dir}/hive-exec-log4j.properties.template"))
-  crt_file(format("{hive_conf_dir}/hive-log4j.properties.template"))
+
+  log4j_exec_filename = 'hive-exec-log4j.properties'
+  if (params.log4j_exec_props != None):
+    PropertiesFile(log4j_exec_filename,
+                   dir=params.hive_conf_dir,
+                   properties=params.log4j_exec_props,
+                   mode=0664,
+                   owner=params.hive_user,
+                   group=params.user_group
+    )
+  elif (os.path.exists("{params.hive_conf_dir}/{log4j_exec_filename}.template")):
+    File(format("{params.hive_conf_dir}/{log4j_exec_filename}"),
+         mode=0644,
+         group=params.user_group,
+         owner=params.hive_user,
+         content=StaticFile(format("{params.hive_conf_dir}/{log4j_exec_filename}.template"))
+    )
+
+  log4j_filename = 'hive-log4j.properties'
+  if (params.log4j_props != None):
+    PropertiesFile(log4j_filename,
+                   dir=params.hive_conf_dir,
+                   properties=params.log4j_props,
+                   mode=0664,
+                   owner=params.hive_user,
+                   group=params.user_group
+    )
+  elif (os.path.exists("{params.hive_conf_dir}/{log4j_filename}.template")):
+    File(format("{params.hive_conf_dir}/{log4j_filename}"),
+         mode=0644,
+         group=params.user_group,
+         owner=params.hive_user,
+         content=StaticFile(format("{params.hive_conf_dir}/{log4j_filename}.template"))
+    )
 
 
 def crt_directory(name):

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/HIVE/package/scripts/params.py

@@ -121,3 +121,15 @@ hcat_pid_dir = status_params.hcat_pid_dir
 hcat_log_dir = config['configurations']['global']['hcat_log_dir']   #hcat_log_dir
 
 hadoop_conf_dir = '/etc/hadoop/conf'
+
+#hive-log4j.properties.template
+if ('hive-log4j' in config['configurations']):
+  log4j_props = config['configurations']['hive-log4j']
+else:
+  log4j_props = None
+
+#hive-exec-log4j.properties.template
+if ('hive-exec-log4j' in config['configurations']):
+  log4j_exec_props = config['configurations']['hive-exec-log4j']
+else:
+  log4j_exec_props = None

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/MAPREDUCE/configuration/mapreduce-log4j.xml

@@ -0,0 +1,73 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+
+  <property>
+    <name>hadoop.mapreduce.jobsummary.logger</name>
+    <value>${hadoop.root.logger}</value>
+  </property>
+
+  <property>
+    <name>hadoop.mapreduce.jobsummary.log.file</name>
+    <value>hadoop-mapreduce.jobsummary.log</value>
+  </property>
+
+  <property>
+    <name>log4j.appender.JSA</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+
+  <property>
+    <name>log4j.appender.JSA.layout.ConversionPattern</name>
+    <value>%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n</value>
+  </property>
+
+  <property>
+    <name>log4j.appender.JSA.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+
+  <property>
+    <name>log4j.appender.JSA.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+
+  <!--Only for HDP1.0 below-->
+  <property>
+    <name>log4j.appender.JSA.File</name>
+    <value>/var/log/hadoop/mapred/${hadoop.mapreduce.jobsummary.log.file}</value>
+  </property>
+
+  <property>
+    <name>log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary</name>
+    <value>${hadoop.mapreduce.jobsummary.logger}</value>
+  </property>
+
+  <property>
+    <name>log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary</name>
+    <value>false</value>
+  </property>
+
+  <!--Only for HDP1.0 above-->
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/MAPREDUCE/metainfo.xml

@@ -95,6 +95,7 @@
         <config-type>global</config-type>
         <config-type>mapred-site</config-type>
         <config-type>mapred-queue-acls</config-type>
+        <config-type>mapreduce-log4j</config-type>
       </configuration-dependencies>
     </service>
 

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/OOZIE/configuration/oozie-log4j.xml

@@ -0,0 +1,183 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+  <property>
+    <name>log4j.appender.oozie</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+
+  <property>
+    <name>log4j.appender.oozie.DatePattern</name>
+    <value>.yyyy-MM-dd-HH</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozie.File</name>
+    <value>${oozie.log.dir}/oozie.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozie.Append</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozie.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozie.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %5p %c{1}:%L - %m%n</value>
+  </property>
+
+  <property>
+    <name>log4j.appender.oozieops</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieops.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieops.File</name>
+    <value>${oozie.log.dir}/oozie-ops.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieops.Append</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieops.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieops.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %5p %c{1}:%L - %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieinstrumentation</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieinstrumentation.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieinstrumentation.File</name>
+    <value>${oozie.log.dir}/oozie-instrumentation.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieinstrumentation.Append</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieinstrumentation.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieinstrumentation.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %5p %c{1}:%L - %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieaudit</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieaudit.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieaudit.File</name>
+    <value>${oozie.log.dir}/oozie-audit.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieaudit.Append</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieaudit.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieaudit.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %5p %c{1}:%L - %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.openjpa</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.openjpa.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.openjpa.File</name>
+    <value>${oozie.log.dir}/oozie-jpa.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.openjpa.Append</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>log4j.appender.openjpa.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.openjpa.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %5p %c{1}:%L - %m%n</value>
+  </property>
+  <property>
+    <name>log4j.logger.openjpa</name>
+    <value>INFO, openjpa</value>
+  </property>
+  <property>
+    <name>log4j.logger.oozieops</name>
+    <value>INFO, oozieops</value>
+  </property>
+  <property>
+    <name>log4j.logger.oozieinstrumentation</name>
+    <value>ALL, oozieinstrumentation</value>
+  </property>
+  <property>
+    <name>log4j.logger.oozieaudit</name>
+    <value>ALL, oozieaudit</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.oozie</name>
+    <value>INFO, oozie</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.hadoop</name>
+    <value>WARN, oozie</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.mortbay</name>
+    <value>WARN, oozie</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.hsqldb</name>
+    <value>WARN, oozie</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.hadoop.security.authentication.server</name>
+    <value>INFO, oozie</value>
+  </property>
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/OOZIE/metainfo.xml

@@ -107,6 +107,7 @@
       <configuration-dependencies>
         <config-type>global</config-type>
         <config-type>oozie-site</config-type>
+        <config-type>oozie-log4j</config-type>
       </configuration-dependencies>
     </service>
   </services>

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/OOZIE/package/scripts/oozie.py

@@ -1,3 +1,4 @@
+import os
 from resource_management import *
 
 def oozie(is_server=False
@@ -20,10 +21,21 @@ def oozie(is_server=False
   TemplateConfig( format("{conf_dir}/oozie-env.sh"),
     owner = params.oozie_user
   )
-  
-  TemplateConfig( format("{conf_dir}/oozie-log4j.properties"),
-    owner = params.oozie_user
-  )
+
+  if (params.log4j_props != None):
+    PropertiesFile('oozie-log4j.properties',
+                   dir=params.conf_dir,
+                   properties=params.log4j_props,
+                   mode=0664,
+                   owner=params.oozie_user,
+                   group=params.user_group,
+    )
+  elif (os.path.exists(format("{params.conf_dir}/oozie-log4j.properties"))):
+    File(format("{params.conf_dir}/oozie-log4j.properties"),
+         mode=0644,
+         group=params.user_group,
+         owner=params.oozie_user
+    )
 
   if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver":
     Execute(format("/bin/sh -c 'cd /usr/lib/ambari-agent/ &&\

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/OOZIE/package/scripts/params.py

@@ -61,4 +61,10 @@ lzo_jar_suffix = "/usr/lib/hadoop/lib/hadoop-lzo-0.5.0.jar" if lzo_enabled else
 if lzo_enabled and jdbc_driver_name:
     jar_path = format("{lzo_jar_suffix}:{jdbc_driver_jar}")        
 else:
-    jar_path = "{lzo_jar_suffix}{jdbc_driver_jar}"
+    jar_path = "{lzo_jar_suffix}{jdbc_driver_jar}"
+
+#oozie-log4j.properties
+if ('oozie-log4j' in config['configurations']):
+  log4j_props = config['configurations']['oozie-log4j']
+else:
+  log4j_props = None

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/OOZIE/package/templates/oozie-log4j.properties.j2

@@ -1,74 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. See accompanying LICENSE file.
-#
-
-# If the Java System property 'oozie.log.dir' is not defined at Oozie start up time
-# XLogService sets its value to '${oozie.home}/logs'
-
-log4j.appender.oozie=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.oozie.DatePattern='.'yyyy-MM-dd-HH
-log4j.appender.oozie.File=${oozie.log.dir}/oozie.log
-log4j.appender.oozie.Append=true
-log4j.appender.oozie.layout=org.apache.log4j.PatternLayout
-log4j.appender.oozie.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n
-
-log4j.appender.oozieops=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.oozieops.DatePattern='.'yyyy-MM-dd
-log4j.appender.oozieops.File=${oozie.log.dir}/oozie-ops.log
-log4j.appender.oozieops.Append=true
-log4j.appender.oozieops.layout=org.apache.log4j.PatternLayout
-log4j.appender.oozieops.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n
-
-log4j.appender.oozieinstrumentation=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.oozieinstrumentation.DatePattern='.'yyyy-MM-dd
-log4j.appender.oozieinstrumentation.File=${oozie.log.dir}/oozie-instrumentation.log
-log4j.appender.oozieinstrumentation.Append=true
-log4j.appender.oozieinstrumentation.layout=org.apache.log4j.PatternLayout
-log4j.appender.oozieinstrumentation.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n
-
-log4j.appender.oozieaudit=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.oozieaudit.DatePattern='.'yyyy-MM-dd
-log4j.appender.oozieaudit.File=${oozie.log.dir}/oozie-audit.log
-log4j.appender.oozieaudit.Append=true
-log4j.appender.oozieaudit.layout=org.apache.log4j.PatternLayout
-log4j.appender.oozieaudit.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n
-
-log4j.appender.openjpa=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.openjpa.DatePattern='.'yyyy-MM-dd
-log4j.appender.openjpa.File=${oozie.log.dir}/oozie-jpa.log
-log4j.appender.openjpa.Append=true
-log4j.appender.openjpa.layout=org.apache.log4j.PatternLayout
-log4j.appender.openjpa.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n
-
-log4j.logger.openjpa=INFO, openjpa
-log4j.logger.oozieops=INFO, oozieops
-log4j.logger.oozieinstrumentation=ALL, oozieinstrumentation
-log4j.logger.oozieaudit=ALL, oozieaudit
-log4j.logger.org.apache.oozie=INFO, oozie
-log4j.logger.org.apache.hadoop=WARN, oozie
-log4j.logger.org.mortbay=WARN, oozie
-log4j.logger.org.hsqldb=WARN, oozie
-log4j.logger.org.apache.hadoop.security.authentication.server=INFO, oozie

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/PIG/configuration/pig-log4j.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+  <property>
+    <name>log4j.logger.org.apache.pig</name>
+    <value>info, A</value>
+  </property>
+  <property>
+    <name>log4j.appender.A</name>
+    <value>org.apache.log4j.ConsoleAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.A.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.A.layout.ConversionPattern</name>
+    <value>%-4r [%t] %-5p %c %x - %m%n</value>
+  </property>
+</configuration>

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/PIG/metainfo.xml

@@ -54,6 +54,7 @@
 
       <configuration-dependencies>
         <config-type>global</config-type>
+        <config-type>pig-log4j</config-type>
       </configuration-dependencies>
 
     </service>

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/PIG/package/scripts/params.py

@@ -33,4 +33,10 @@ user_group = config['configurations']['global']['user_group']
 
 # not supporting 32 bit jdk.
 java64_home = config['hostLevelParams']['java_home']
-hadoop_home = "/usr"
+hadoop_home = "/usr"
+
+#log4j.properties
+if ('pig-log4j' in config['configurations']):
+  log4j_props = config['configurations']['pig-log4j']
+else:
+  log4j_props = None

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/PIG/package/scripts/pig.py

@@ -19,6 +19,7 @@ limitations under the License.
 Ambari Agent
 
 """
+import os
 
 from resource_management import *
 
@@ -30,7 +31,22 @@ def pig():
     group = params.user_group
   )
 
-  pig_TemplateConfig( ['pig-env.sh','pig.properties','log4j.properties'])
+  pig_TemplateConfig( ['pig-env.sh','pig.properties'])
+
+  if (params.log4j_props != None):
+    PropertiesFile('log4j.properties',
+                   dir=params.pig_conf_dir,
+                   properties=params.log4j_props,
+                   mode=0664,
+                   owner=params.hdfs_user,
+                   group=params.user_group,
+    )
+  elif (os.path.exists(format("{params.pig_conf_dir}/log4j.properties"))):
+    File(format("{params.pig_conf_dir}/log4j.properties"),
+         mode=0644,
+         group=params.user_group,
+         owner=params.hdfs_user
+    )
   
   
 def pig_TemplateConfig(name):
@@ -43,4 +59,4 @@ def pig_TemplateConfig(name):
     TemplateConfig( format("{pig_conf_dir}/{x}"),
         owner = params.hdfs_user
     )
-  
+  

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/PIG/package/templates/log4j.properties.j2

@@ -1,30 +0,0 @@
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-#
-
-# ***** Set root logger level to DEBUG and its only appender to A.
-log4j.logger.org.apache.pig=info, A
-
-# ***** A is set to be a ConsoleAppender.
-log4j.appender.A=org.apache.log4j.ConsoleAppender
-# ***** A uses PatternLayout.
-log4j.appender.A.layout=org.apache.log4j.PatternLayout
-log4j.appender.A.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/ZOOKEEPER/configuration/zookeeper-log4j.xml

@@ -0,0 +1,84 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+  <property>
+    <name>log4j.appender.CONSOLE</name>
+    <value>org.apache.log4j.ConsoleAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.CONSOLE.Threshold</name>
+    <value>INFO</value>
+  </property>
+  <property>
+    <name>log4j.appender.CONSOLE.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.CONSOLE.layout.ConversionPattern</name>
+    <value>%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.ROLLINGFILE</name>
+    <value>org.apache.log4j.RollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.ROLLINGFILE.Threshold</name>
+    <value>DEBUG</value>
+  </property>
+  <property>
+    <name>log4j.appender.ROLLINGFILE.File</name>
+    <value>zookeeper.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.ROLLINGFILE.MaxFileSize</name>
+    <value>10MB</value>
+  </property>
+  <property>
+    <name>log4j.appender.ROLLINGFILE.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.ROLLINGFILE.layout.ConversionPattern</name>
+    <value>%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.TRACEFILE</name>
+    <value>org.apache.log4j.FileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.TRACEFILE.Threshold</name>
+    <value>TRACE</value>
+  </property>
+  <property>
+    <name>log4j.appender.TRACEFILE.File</name>
+    <value>zookeeper_trace.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.TRACEFILE.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.TRACEFILE.layout.ConversionPattern</name>
+    <value>%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n</value>
+  </property>
+</configuration>

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/ZOOKEEPER/metainfo.xml

@@ -66,6 +66,7 @@
 
       <configuration-dependencies>
         <config-type>global</config-type>
+        <config-type>zookeeper-log4j</config-type>
       </configuration-dependencies>
     </service>
   </services>

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/ZOOKEEPER/package/scripts/params.py

@@ -69,3 +69,9 @@ security_enabled = config['configurations']['global']['security_enabled']
 smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
 smokeuser = config['configurations']['global']['smokeuser']
 kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+
+#log4j.properties
+if ('zookeeper-log4j' in config['configurations']):
+  log4j_props = config['configurations']['zookeeper-log4j']
+else:
+  log4j_props = None

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/ZOOKEEPER/package/scripts/zookeeper.py

@@ -19,6 +19,7 @@ limitations under the License.
 Ambari Agent
 
 """
+import os
 
 from resource_management import *
 import sys
@@ -63,7 +64,20 @@ def zookeeper(type = None):
          content = myid
     )
 
-  configFile("log4j.properties", template_name="log4j.properties.j2")
+  if (params.log4j_props != None):
+    PropertiesFile('log4j.properties',
+      dir=params.config_dir,
+      properties=params.log4j_props,
+      mode=0664,
+      owner=params.zk_user,
+      group=params.user_group,
+    )
+  elif (os.path.exists(format("{params.config_dir}/log4j.properties"))):
+    File(format("{params.config_dir}/log4j.properties"),
+      mode=0644,
+      group=params.user_group,
+      owner=params.zk_user
+    )
 
   if params.security_enabled:
     if type == "server":

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/ZOOKEEPER/package/templates/log4j.properties.j2

@@ -1,71 +0,0 @@
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-#
-
-#
-# ZooKeeper Logging Configuration
-#
-
-# Format is "<default threshold> (, <appender>)+
-
-# DEFAULT: console appender only
-log4j.rootLogger=INFO, CONSOLE
-
-# Example with rolling log file
-#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE
-
-# Example with rolling log file and tracing
-#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE
-
-#
-# Log INFO level and above messages to the console
-#
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.Threshold=INFO
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n
-
-#
-# Add ROLLINGFILE to rootLogger to get log file output
-#    Log DEBUG level and above messages to a log file
-log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender
-log4j.appender.ROLLINGFILE.Threshold=DEBUG
-log4j.appender.ROLLINGFILE.File=zookeeper.log
-
-# Max log file size of 10MB
-log4j.appender.ROLLINGFILE.MaxFileSize=10MB
-# uncomment the next line to limit number of backup files
-#log4j.appender.ROLLINGFILE.MaxBackupIndex=10
-
-log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout
-log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n
-
-
-#
-# Add TRACEFILE to rootLogger to get log file output
-#    Log DEBUG level and above messages to a log file
-log4j.appender.TRACEFILE=org.apache.log4j.FileAppender
-log4j.appender.TRACEFILE.Threshold=TRACE
-log4j.appender.TRACEFILE.File=zookeeper_trace.log
-
-log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout
-### Notice we are including log4j's NDC here (%x)
-log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n

ambari-server/src/main/resources/stacks/HDP/2.1.1/hooks/before-START/scripts/params.py

@@ -170,3 +170,28 @@ if dfs_ha_enabled:
   namenode_id = None
 
 dfs_hosts = default('/configurations/hdfs-site/dfs.hosts', None)
+
+#log4j.properties
+rca_property_map = {
+  'ambari.jobhistory.database': ambari_db_rca_url,
+  'ambari.jobhistory.driver': ambari_db_rca_driver,
+  'ambari.jobhistory.user': ambari_db_rca_username,
+  'ambari.jobhistory.password': ambari_db_rca_password,
+  'ambari.jobhistory.logger': 'DEBUG,JHA',
+
+  'log4j.appender.JHA': 'org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender',
+  'log4j.appender.JHA.database': '${ambari.jobhistory.database}',
+  'log4j.appender.JHA.driver': '${ambari.jobhistory.driver}',
+  'log4j.appender.JHA.user': '${ambari.jobhistory.user}',
+  'log4j.appender.JHA.password': '${ambari.jobhistory.password}',
+
+  'log4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger': '${ambari.jobhistory.logger}',
+  'log4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger': 'true'
+}
+
+if ('hdfs-log4j' in config['configurations']):
+  log4j_props = config['configurations']['hdfs-log4j']
+  if 'yarn-log4j' in config['configurations']:
+    log4j_props.update(config['configurations']['yarn-log4j'])
+else:
+  log4j_props = None

ambari-server/src/main/resources/stacks/HDP/2.1.1/hooks/before-START/scripts/shared_initialization.py

@@ -145,12 +145,22 @@ def setup_hadoop():
        content=Template(health_check_template + ".j2")
   )
 
-  File(os.path.join(params.hadoop_conf_dir, "log4j.properties"),
-       owner=params.hdfs_user,
-       content=Template("log4j.properties.j2")
-  )
+  log4j_filename = os.path.join(params.hadoop_conf_dir, "log4j.properties")
+  if (params.log4j_props != None):
+    PropertiesFile(log4j_filename,
+                   properties=params.log4j_props,
+                   mode=0664,
+                   owner=params.hdfs_user,
+                   group=params.user_group,
+    )
+  elif (os.path.exists(format("{params.hadoop_conf_dir}/log4j.properties"))):
+    File(log4j_filename,
+         mode=0644,
+         group=params.user_group,
+         owner=params.hdfs_user,
+    )
 
-  update_log4j_props(os.path.join(params.hadoop_conf_dir, "log4j.properties"))
+  update_log4j_props(log4j_filename)
 
   File(os.path.join(params.hadoop_conf_dir, "hadoop-metrics2.properties"),
        owner=params.hdfs_user,
@@ -275,26 +285,11 @@ def setup_configs():
 def update_log4j_props(file):
   import params
 
-  property_map = {
-    'ambari.jobhistory.database': params.ambari_db_rca_url,
-    'ambari.jobhistory.driver': params.ambari_db_rca_driver,
-    'ambari.jobhistory.user': params.ambari_db_rca_username,
-    'ambari.jobhistory.password': params.ambari_db_rca_password,
-    'ambari.jobhistory.logger': 'DEBUG,JHA',
-
-    'log4j.appender.JHA': 'org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender',
-    'log4j.appender.JHA.database': '${ambari.jobhistory.database}',
-    'log4j.appender.JHA.driver': '${ambari.jobhistory.driver}',
-    'log4j.appender.JHA.user': '${ambari.jobhistory.user}',
-    'log4j.appender.JHA.password': '${ambari.jobhistory.password}',
-
-    'log4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger': '${ambari.jobhistory.logger}',
-    'log4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger': 'true'
-  }
-  for key in property_map:
-    value = property_map[key]
+  for key in params.rca_property_map:
+    value = params.rca_property_map[key]
     Execute(format(
-      "sed -i 's~\\({rca_disabled_prefix}\\)\\?{key}=.*~{rca_prefix}{key}={value}~' {file}"))
+      "sed -i 's~\\({rca_disabled_prefix}\\)\\?{key}=.*~{rca_prefix}{key}={value}~' {file}")
+    )
 
 
 def generate_include_file():

ambari-server/src/main/resources/stacks/HDP/2.1.1/hooks/before-START/templates/log4j.properties.j2

@@ -1,218 +0,0 @@
-# Copyright 2011 The Apache Software Foundation
-# 
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Define some default values that can be overridden by system properties
-hadoop.root.logger=INFO,console
-hadoop.log.dir=.
-hadoop.log.file=hadoop.log
-
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hadoop.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshhold=ALL
-
-#
-# Daily Rolling File Appender
-#
-
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
-
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-# 30-day backup
-#log4j.appender.DRFA.MaxBackupIndex=30
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this 
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-
-#
-# TaskLog Appender
-#
-
-#Default values
-hadoop.tasklog.taskid=null
-hadoop.tasklog.iscleanup=false
-hadoop.tasklog.noKeepSplits=4
-hadoop.tasklog.totalLogFileSize=100
-hadoop.tasklog.purgeLogSplits=true
-hadoop.tasklog.logsRetainHours=12
-
-log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
-log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
-log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}
-log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
-
-log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
-log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-
-#
-#Security audit appender
-#
-hadoop.security.logger=INFO,console
-hadoop.security.log.maxfilesize=256MB
-hadoop.security.log.maxbackupindex=20
-log4j.category.SecurityLogger=${hadoop.security.logger}
-hadoop.security.log.file=SecurityAuth.audit
-log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender 
-log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
-log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
-log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd
-
-log4j.appender.RFAS=org.apache.log4j.RollingFileAppender 
-log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
-log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}
-log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}
-
-#
-# hdfs audit logging
-#
-hdfs.audit.logger=INFO,console
-log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}
-log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false
-log4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log
-log4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd
-
-#
-# mapred audit logging
-#
-mapred.audit.logger=INFO,console
-log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}
-log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false
-log4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log
-log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd
-
-#
-# Rolling File Appender
-#
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
-
-# Logfile size and and 30-day backups
-log4j.appender.RFA.MaxFileSize=256MB
-log4j.appender.RFA.MaxBackupIndex=10
-
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-# Custom Logging levels
-
-hadoop.metrics.log.level=INFO
-#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
-#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
-#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
-log4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}
-
-# Jets3t library
-log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
-
-#
-# Null Appender
-# Trap security logger on the hadoop client side
-#
-log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
-
-{% if is_jtnode_master or is_rmnode_master %}
-#
-# Job Summary Appender 
-#
-# Use following logger to send summary to separate file defined by 
-# hadoop.mapreduce.jobsummary.log.file rolled daily:
-# hadoop.mapreduce.jobsummary.logger=INFO,JSA
-# 
-hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
-hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
-log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender
-# Set the ResourceManager summary log filename
-yarn.server.resourcemanager.appsummary.log.file=hadoop-mapreduce.jobsummary.log
-# Set the ResourceManager summary log level and appender
-yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger}
-#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY
-
-# To enable AppSummaryLogging for the RM,
-# set yarn.server.resourcemanager.appsummary.logger to
-# <LEVEL>,RMSUMMARY in hadoop-env.sh
-
-# Appender for ResourceManager Application Summary Log
-# Requires the following properties to be set
-#    - hadoop.log.dir (Hadoop Log directory)
-#    - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename)
-#    - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender)
-log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender
-log4j.appender.RMSUMMARY.File={{yarn_log_dir_prefix}}/{{yarn_user}}/${yarn.server.resourcemanager.appsummary.log.file}
-log4j.appender.RMSUMMARY.MaxFileSize=256MB
-log4j.appender.RMSUMMARY.MaxBackupIndex=20
-log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout
-log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
-log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-log4j.appender.JSA.DatePattern=.yyyy-MM-dd
-log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
-log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger}
-log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false
-{% endif %}
-
-{{rca_prefix}}ambari.jobhistory.database={{ambari_db_rca_url}}
-{{rca_prefix}}ambari.jobhistory.driver={{ambari_db_rca_driver}}
-{{rca_prefix}}ambari.jobhistory.user={{ambari_db_rca_username}}
-{{rca_prefix}}ambari.jobhistory.password={{ambari_db_rca_password}}
-{{rca_prefix}}ambari.jobhistory.logger=DEBUG,JHA
-
-{{rca_prefix}}log4j.appender.JHA=org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender
-{{rca_prefix}}log4j.appender.JHA.database=${ambari.jobhistory.database}
-{{rca_prefix}}log4j.appender.JHA.driver=${ambari.jobhistory.driver}
-{{rca_prefix}}log4j.appender.JHA.user=${ambari.jobhistory.user}
-{{rca_prefix}}log4j.appender.JHA.password=${ambari.jobhistory.password}
-
-{{rca_prefix}}log4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=${ambari.jobhistory.logger}
-{{rca_prefix}}log4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=true

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HBASE/configuration/hbase-log4j.xml

@@ -0,0 +1,183 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+
+  <property>
+    <name>hbase.root.logger</name>
+    <value>INFO,console</value>
+  </property>
+  <property>
+    <name>hbase.security.logger</name>
+    <value>INFO,console</value>
+  </property>
+  <property>
+    <name>hbase.log.dir</name>
+    <value>.</value>
+  </property>
+  <property>
+    <name>hbase.log.file</name>
+    <value>hbase.log</value>
+  </property>
+  <property>
+    <name>log4j.rootLogger</name>
+    <value>${hbase.root.logger}</value>
+  </property>
+  <property>
+    <name>log4j.threshold</name>
+    <value>ALL</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.File</name>
+    <value>${hbase.log.dir}/${hbase.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %-5p [%t] %c{2}: %m%n</value>
+  </property>
+  <property>
+    <name>hbase.log.maxfilesize</name>
+    <value>256MB</value>
+  </property>
+  <property>
+    <name>hbase.log.maxbackupindex</name>
+    <value>20</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA</name>
+    <value>org.apache.log4j.RollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.File</name>
+    <value>${hbase.log.dir}/${hbase.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.MaxFileSize</name>
+    <value>${hbase.log.maxfilesize}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.MaxBackupIndex</name>
+    <value>${hbase.log.maxbackupindex}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %-5p [%t] %c{2}: %m%n</value>
+  </property>
+  <property>
+    <name>hbase.security.log.file</name>
+    <value>SecurityAuth.audit</value>
+  </property>
+  <property>
+    <name>hbase.security.log.maxfilesize</name>
+    <value>256MB</value>
+  </property>
+  <property>
+    <name>hbase.security.log.maxbackupindex</name>
+    <value>20</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS</name>
+    <value>org.apache.log4j.RollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.File</name>
+    <value>${hbase.log.dir}/${hbase.security.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.MaxFileSize</name>
+    <value>${hbase.security.log.maxfilesize}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.MaxBackupIndex</name>
+    <value>${hbase.security.log.maxbackupindex}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %p %c: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.category.SecurityLogger</name>
+    <value>${hbase.security.logger}</value>
+  </property>
+  <property>
+    <name>log4j.additivity.SecurityLogger</name>
+    <value>false</value>
+  </property>
+  <property>
+    <name>log4j.appender.NullAppender</name>
+    <value>org.apache.log4j.varia.NullAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.console</name>
+    <value>org.apache.log4j.ConsoleAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.target</name>
+    <value>System.err</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %-5p [%t] %c{2}: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.zookeeper</name>
+    <value>INFO</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.hadoop.hbase</name>
+    <value>DEBUG</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil</name>
+    <value>INFO</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher</name>
+    <value>INFO</value>
+  </property>
+
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HBASE/metainfo.xml

@@ -86,6 +86,7 @@
         <config-type>global</config-type>
         <config-type>hbase-policy</config-type>
         <config-type>hbase-site</config-type>
+        <config-type>hbase-log4j</config-type>
       </configuration-dependencies>
 
     </service>

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HBASE/package/scripts/hbase.py

@@ -17,6 +17,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 
 """
+import os
 
 from resource_management import *
 import sys
@@ -78,7 +79,22 @@ def hbase(type=None # 'master' or 'regionserver' or 'client'
     Directory ( [params.tmp_dir, params.log_dir],
       owner = params.hbase_user,
       recursive = True
-    )    
+    )
+
+  if (params.log4j_props != None):
+    PropertiesFile('log4j.properties',
+      dir=params.conf_dir,
+      properties=params.log4j_props,
+      mode=0664,
+      owner=params.hbase_user,
+      group=params.user_group,
+    )
+  elif (os.path.exists(format("{params.conf_dir}/log4j.properties"))):
+    File(format("{params.conf_dir}/log4j.properties"),
+      mode=0644,
+      group=params.user_group,
+      owner=params.hbase_user
+    )
 
 def hbase_TemplateConfig(name, 
                          tag=None

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HBASE/package/scripts/params.py

@@ -82,3 +82,9 @@ regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regions
 smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
 hbase_user_keytab = config['configurations']['global']['hbase_user_keytab']
 kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+
+#log4j.properties
+if ('hbase-log4j' in config['configurations']):
+  log4j_props = config['configurations']['hbase-log4j']
+else:
+  log4j_props = None

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HDFS/configuration/hdfs-log4j.xml

@@ -0,0 +1,305 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+  <property>
+    <name>hadoop.root.logger</name>
+    <value>INFO,console</value>
+  </property>
+  <property>
+    <name>hadoop.log.dir</name>
+    <value>.</value>
+  </property>
+  <property>
+    <name>hadoop.log.file</name>
+    <value>hadoop.log</value>
+  </property>
+  <property>
+    <name>log4j.rootLogger</name>
+    <value>${hadoop.root.logger}, EventCounter</value>
+  </property>
+  <property>
+    <name>log4j.threshhold</name>
+    <value>ALL</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.File</name>
+    <value>${hadoop.log.dir}/${hadoop.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %p %c: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.console</name>
+    <value>org.apache.log4j.ConsoleAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.target</name>
+    <value>System.err</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.layout.ConversionPattern</name>
+    <value>%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n</value>
+  </property>
+  <property>
+    <name>hadoop.tasklog.taskid</name>
+    <value>null</value>
+  </property>
+  <property>
+    <name>hadoop.tasklog.iscleanup</name>
+    <value>false</value>
+  </property>
+  <property>
+    <name>hadoop.tasklog.noKeepSplits</name>
+    <value>4</value>
+  </property>
+  <property>
+    <name>hadoop.tasklog.totalLogFileSize</name>
+    <value>100</value>
+  </property>
+  <property>
+    <name>hadoop.tasklog.purgeLogSplits</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>hadoop.tasklog.logsRetainHours</name>
+    <value>12</value>
+  </property>
+  <property>
+    <name>log4j.appender.TLA</name>
+    <value>org.apache.hadoop.mapred.TaskLogAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.TLA.taskId</name>
+    <value>${hadoop.tasklog.taskid}</value>
+  </property>
+  <property>
+    <name>log4j.appender.TLA.isCleanup</name>
+    <value>${hadoop.tasklog.iscleanup}</value>
+  </property>
+  <property>
+    <name>log4j.appender.TLA.totalLogFileSize</name>
+    <value>${hadoop.tasklog.totalLogFileSize}</value>
+  </property>
+  <property>
+    <name>log4j.appender.TLA.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.TLA.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %p %c: %m%n</value>
+  </property>
+  <property>
+    <name>hadoop.security.logger</name>
+    <value>INFO,console</value>
+  </property>
+  <property>
+    <name>hadoop.security.log.maxfilesize</name>
+    <value>256MB</value>
+  </property>
+  <property>
+    <name>hadoop.security.log.maxbackupindex</name>
+    <value>20</value>
+  </property>
+  <property>
+    <name>log4j.category.SecurityLogger</name>
+    <value>${hadoop.security.logger}</value>
+  </property>
+  <property>
+    <name>hadoop.security.log.file</name>
+    <value>SecurityAuth.audit</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAS</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAS.File</name>
+    <value>${hadoop.log.dir}/${hadoop.security.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAS.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAS.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %p %c: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAS.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS</name>
+    <value>org.apache.log4j.RollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.File</name>
+    <value>${hadoop.log.dir}/${hadoop.security.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %p %c: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.MaxFileSize</name>
+    <value>${hadoop.security.log.maxfilesize}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFAS.MaxBackupIndex</name>
+    <value>${hadoop.security.log.maxbackupindex}</value>
+  </property>
+  <property>
+    <name>hdfs.audit.logger</name>
+    <value>INFO,console</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit</name>
+    <value>${hdfs.audit.logger}</value>
+  </property>
+  <property>
+    <name>log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit</name>
+    <value>false</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAAUDIT</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAAUDIT.File</name>
+    <value>${hadoop.log.dir}/hdfs-audit.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAAUDIT.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAAUDIT.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %p %c{2}: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFAAUDIT.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>mapred.audit.logger</name>
+    <value>INFO,console</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.hadoop.mapred.AuditLogger</name>
+    <value>${mapred.audit.logger}</value>
+  </property>
+  <property>
+    <name>log4j.additivity.org.apache.hadoop.mapred.AuditLogger</name>
+    <value>false</value>
+  </property>
+  <property>
+    <name>log4j.appender.MRAUDIT</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.MRAUDIT.File</name>
+    <value>${hadoop.log.dir}/mapred-audit.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.MRAUDIT.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.MRAUDIT.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %p %c{2}: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.MRAUDIT.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA</name>
+    <value>org.apache.log4j.RollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.File</name>
+    <value>${hadoop.log.dir}/${hadoop.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.MaxFileSize</name>
+    <value>256MB</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.MaxBackupIndex</name>
+    <value>10</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %-5p %c{2} - %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.RFA.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n</value>
+  </property>
+  <property>
+    <name>hadoop.metrics.log.level</name>
+    <value>INFO</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.hadoop.metrics2</name>
+    <value>${hadoop.metrics.log.level}</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service</name>
+    <value>ERROR</value>
+  </property>
+  <property>
+    <name>log4j.appender.NullAppender</name>
+    <value>org.apache.log4j.varia.NullAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.EventCounter</name>
+    <value>org.apache.hadoop.log.metrics.EventCounter</value>
+  </property>
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HDFS/metainfo.xml

@@ -146,6 +146,7 @@
         <config-type>global</config-type>
         <config-type>hdfs-site</config-type>
         <config-type>hadoop-policy</config-type>
+        <config-type>hdfs-log4j</config-type>
       </configuration-dependencies>
     </service>
   </services>

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/configuration/hive-exec-log4j.xml

@@ -0,0 +1,122 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+
+  <property>
+    <name>hive.log.threshold</name>
+    <value>ALL</value>
+  </property>
+  <property>
+    <name>hive.root.logger</name>
+    <value>INFO,FA</value>
+  </property>
+  <property>
+    <name>hive.log.dir</name>
+    <value>/tmp/${user.name}</value>
+  </property>
+  <property>
+    <name>hive.log.file</name>
+    <value>${hive.query.id}.log</value>
+  </property>
+  <property>
+    <name>log4j.rootLogger</name>
+    <value>${hive.root.logger}, EventCounter</value>
+  </property>
+  <property>
+    <name>log4j.threshhold</name>
+    <value>${hive.log.threshold}</value>
+  </property>
+  <property>
+    <name>log4j.appender.FA</name>
+    <value>org.apache.log4j.FileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.FA.File</name>
+    <value>${hive.log.dir}/${hive.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.FA.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.FA.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.console</name>
+    <value>org.apache.log4j.ConsoleAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.target</name>
+    <value>System.err</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.layout.ConversionPattern</name>
+    <value>%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.EventCounter</name>
+    <value>org.apache.hadoop.hive.shims.HiveEventCounter</value>
+  </property>
+  <property>
+    <name>log4j.category.DataNucleus</name>
+    <value>ERROR,FA</value>
+  </property>
+  <property>
+    <name>log4j.category.Datastore</name>
+    <value>ERROR,FA</value>
+  </property>
+  <property>
+    <name>log4j.category.Datastore.Schema</name>
+    <value>ERROR,FA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.Datastore</name>
+    <value>ERROR,FA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.Plugin</name>
+    <value>ERROR,FA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.MetaData</name>
+    <value>ERROR,FA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.Query</name>
+    <value>ERROR,FA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.General</name>
+    <value>ERROR,FA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.Enhancer</name>
+    <value>ERROR,FA</value>
+  </property>
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/configuration/hive-log4j.xml

@@ -0,0 +1,130 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+
+  <property>
+    <name>hive.log.threshold</name>
+    <value>ALL</value>
+  </property>
+  <property>
+    <name>hive.root.logger</name>
+    <value>WARN,DRFA</value>
+  </property>
+  <property>
+    <name>hive.log.dir</name>
+    <value>/tmp/${user.name}</value>
+  </property>
+  <property>
+    <name>hive.log.file</name>
+    <value>hive.log</value>
+  </property>
+  <property>
+    <name>log4j.rootLogger</name>
+    <value>${hive.root.logger}, EventCounter</value>
+  </property>
+  <property>
+    <name>log4j.threshold</name>
+    <value>${hive.log.threshold}</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.File</name>
+    <value>${hive.log.dir}/${hive.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.DRFA.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.console</name>
+    <value>org.apache.log4j.ConsoleAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.target</name>
+    <value>System.err</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.layout.ConversionPattern</name>
+    <value>%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.console.encoding</name>
+    <value>UTF-8</value>
+  </property>
+  <property>
+    <name>log4j.appender.EventCounter</name>
+    <value>org.apache.hadoop.hive.shims.HiveEventCounter</value>
+  </property>
+  <property>
+    <name>log4j.category.DataNucleus</name>
+    <value>ERROR,DRFA</value>
+  </property>
+  <property>
+    <name>log4j.category.Datastore</name>
+    <value>ERROR,DRFA</value>
+  </property>
+  <property>
+    <name>log4j.category.Datastore.Schema</name>
+    <value>ERROR,DRFA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.Datastore</name>
+    <value>ERROR,DRFA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.Plugin</name>
+    <value>ERROR,DRFA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.MetaData</name>
+    <value>ERROR,DRFA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.Query</name>
+    <value>ERROR,DRFA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.General</name>
+    <value>ERROR,DRFA</value>
+  </property>
+  <property>
+    <name>log4j.category.JPOX.Enhancer</name>
+    <value>ERROR,DRFA</value>
+  </property>
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/metainfo.xml

@@ -117,6 +117,8 @@
 
       <configuration-dependencies>
         <config-type>hive-site</config-type>
+        <config-type>hive-log4j</config-type>
+        <config-type>hive-exec-log4j</config-type>
       </configuration-dependencies>
     </service>
 

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/package/scripts/hive.py

@@ -20,6 +20,7 @@ limitations under the License.
 
 from resource_management import *
 import sys
+import os
 
 
 def hive(name=None):
@@ -78,8 +79,40 @@ def hive(name=None):
 
   crt_file(format("{hive_conf_dir}/hive-default.xml.template"))
   crt_file(format("{hive_conf_dir}/hive-env.sh.template"))
-  crt_file(format("{hive_conf_dir}/hive-exec-log4j.properties.template"))
-  crt_file(format("{hive_conf_dir}/hive-log4j.properties.template"))
+
+  log4j_exec_filename = 'hive-exec-log4j.properties'
+  if (params.log4j_exec_props != None):
+    PropertiesFile(log4j_exec_filename,
+                   dir=params.hive_conf_dir,
+                   properties=params.log4j_exec_props,
+                   mode=0664,
+                   owner=params.hive_user,
+                   group=params.user_group
+    )
+  elif (os.path.exists("{params.hive_conf_dir}/{log4j_exec_filename}.template")):
+    File(format("{params.hive_conf_dir}/{log4j_exec_filename}"),
+         mode=0644,
+         group=params.user_group,
+         owner=params.hive_user,
+         content=StaticFile(format("{params.hive_conf_dir}/{log4j_exec_filename}.template"))
+    )
+
+  log4j_filename = 'hive-log4j.properties'
+  if (params.log4j_props != None):
+    PropertiesFile(log4j_filename,
+                   dir=params.hive_conf_dir,
+                   properties=params.log4j_props,
+                   mode=0664,
+                   owner=params.hive_user,
+                   group=params.user_group
+    )
+  elif (os.path.exists("{params.hive_conf_dir}/{log4j_filename}.template")):
+    File(format("{params.hive_conf_dir}/{log4j_filename}"),
+         mode=0644,
+         group=params.user_group,
+         owner=params.hive_user,
+         content=StaticFile(format("{params.hive_conf_dir}/{log4j_filename}.template"))
+    )
 
 
 def crt_directory(name):

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/package/scripts/params.py

@@ -121,3 +121,15 @@ hcat_pid_dir = status_params.hcat_pid_dir
 hcat_log_dir = config['configurations']['global']['hcat_log_dir']   #hcat_log_dir
 
 hadoop_conf_dir = '/etc/hadoop/conf'
+
+#hive-log4j.properties.template
+if ('hive-log4j' in config['configurations']):
+  log4j_props = config['configurations']['hive-log4j']
+else:
+  log4j_props = None
+
+#hive-exec-log4j.properties.template
+if ('hive-exec-log4j' in config['configurations']):
+  log4j_exec_props = config['configurations']['hive-exec-log4j']
+else:
+  log4j_exec_props = None

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/OOZIE/configuration/oozie-log4j.xml

@@ -0,0 +1,183 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+  <property>
+    <name>log4j.appender.oozie</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+
+  <property>
+    <name>log4j.appender.oozie.DatePattern</name>
+    <value>.yyyy-MM-dd-HH</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozie.File</name>
+    <value>${oozie.log.dir}/oozie.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozie.Append</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozie.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozie.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %5p %c{1}:%L - %m%n</value>
+  </property>
+
+  <property>
+    <name>log4j.appender.oozieops</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieops.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieops.File</name>
+    <value>${oozie.log.dir}/oozie-ops.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieops.Append</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieops.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieops.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %5p %c{1}:%L - %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieinstrumentation</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieinstrumentation.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieinstrumentation.File</name>
+    <value>${oozie.log.dir}/oozie-instrumentation.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieinstrumentation.Append</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieinstrumentation.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieinstrumentation.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %5p %c{1}:%L - %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieaudit</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieaudit.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieaudit.File</name>
+    <value>${oozie.log.dir}/oozie-audit.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieaudit.Append</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieaudit.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.oozieaudit.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %5p %c{1}:%L - %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.openjpa</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.openjpa.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.openjpa.File</name>
+    <value>${oozie.log.dir}/oozie-jpa.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.openjpa.Append</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>log4j.appender.openjpa.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.openjpa.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %5p %c{1}:%L - %m%n</value>
+  </property>
+  <property>
+    <name>log4j.logger.openjpa</name>
+    <value>INFO, openjpa</value>
+  </property>
+  <property>
+    <name>log4j.logger.oozieops</name>
+    <value>INFO, oozieops</value>
+  </property>
+  <property>
+    <name>log4j.logger.oozieinstrumentation</name>
+    <value>ALL, oozieinstrumentation</value>
+  </property>
+  <property>
+    <name>log4j.logger.oozieaudit</name>
+    <value>ALL, oozieaudit</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.oozie</name>
+    <value>INFO, oozie</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.hadoop</name>
+    <value>WARN, oozie</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.mortbay</name>
+    <value>WARN, oozie</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.hsqldb</name>
+    <value>WARN, oozie</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.hadoop.security.authentication.server</name>
+    <value>INFO, oozie</value>
+  </property>
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/OOZIE/metainfo.xml

@@ -78,6 +78,7 @@
       <configuration-dependencies>
         <config-type>global</config-type>
         <config-type>oozie-site</config-type>
+        <config-type>oozie-log4j</config-type>
       </configuration-dependencies>
     </service>
   </services>

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/OOZIE/package/scripts/oozie.py

@@ -17,6 +17,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 
 """
+import os
 
 from resource_management import *
 
@@ -41,10 +42,21 @@ def oozie(is_server=False # TODO: see if see can remove this
   TemplateConfig( format("{conf_dir}/oozie-env.sh"),
     owner = params.oozie_user
   )
-  
-  TemplateConfig( format("{conf_dir}/oozie-log4j.properties"),
-    owner = params.oozie_user
-  )
+
+  if (params.log4j_props != None):
+    PropertiesFile('oozie-log4j.properties',
+      dir=params.conf_dir,
+      properties=params.log4j_props,
+      mode=0664,
+      owner=params.oozie_user,
+      group=params.user_group,
+    )
+  elif (os.path.exists(format("{params.conf_dir}/oozie-log4j.properties"))):
+    File(format("{params.conf_dir}/oozie-log4j.properties"),
+      mode=0644,
+      group=params.user_group,
+      owner=params.oozie_user
+    )
 
   if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver":
     Execute(format("/bin/sh -c 'cd /usr/lib/ambari-agent/ &&\

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/OOZIE/package/scripts/params.py

@@ -76,4 +76,8 @@ falcon_host = default("/clusterHostInfo/falcon_server_hosts", [])
 has_falcon_host = not len(falcon_host)  == 0
 falcon_home = '/usr/lib/falcon'
 
-
+#oozie-log4j.properties
+if ('oozie-log4j' in config['configurations']):
+  log4j_props = config['configurations']['oozie-log4j']
+else:
+  log4j_props = None

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/PIG/configuration/pig-log4j.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+  <property>
+    <name>log4j.logger.org.apache.pig</name>
+    <value>info, A</value>
+  </property>
+  <property>
+    <name>log4j.appender.A</name>
+    <value>org.apache.log4j.ConsoleAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.A.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.A.layout.ConversionPattern</name>
+    <value>%-4r [%t] %-5p %c %x - %m%n</value>
+  </property>
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/PIG/metainfo.xml

@@ -53,6 +53,7 @@
 
       <configuration-dependencies>
         <config-type>global</config-type>
+        <config-type>pig-log4j</config-type>
       </configuration-dependencies>
 
     </service>

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/PIG/package/scripts/params.py

@@ -33,4 +33,10 @@ user_group = config['configurations']['global']['user_group']
 
 # not supporting 32 bit jdk.
 java64_home = config['hostLevelParams']['java_home']
-hadoop_home = "/usr"
+hadoop_home = "/usr"
+
+#log4j.properties
+if ('pig-log4j' in config['configurations']):
+  log4j_props = config['configurations']['pig-log4j']
+else:
+  log4j_props = None

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/PIG/package/scripts/pig.py

@@ -19,6 +19,7 @@ limitations under the License.
 Ambari Agent
 
 """
+import os
 
 from resource_management import *
 
@@ -30,9 +31,23 @@ def pig():
     group = params.user_group
   )
 
-  pig_TemplateConfig( ['pig-env.sh','pig.properties','log4j.properties'])
-  
-  
+  pig_TemplateConfig( ['pig-env.sh','pig.properties'])
+
+  if (params.log4j_props != None):
+    PropertiesFile('log4j.properties',
+      dir=params.pig_conf_dir,
+      properties=params.log4j_props,
+      mode=0664,
+      owner=params.hdfs_user,
+      group=params.user_group,
+    )
+  elif (os.path.exists(format("{params.pig_conf_dir}/log4j.properties"))):
+    File(format("{params.pig_conf_dir}/log4j.properties"),
+      mode=0644,
+      group=params.user_group,
+      owner=params.hdfs_user
+    )
+
 def pig_TemplateConfig(name):
   import params
   
@@ -43,4 +58,4 @@ def pig_TemplateConfig(name):
     TemplateConfig( format("{pig_conf_dir}/{x}"),
         owner = params.hdfs_user
     )
-  
+  

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/PIG/package/templates/log4j.properties.j2

@@ -1,30 +0,0 @@
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-#
-
-# ***** Set root logger level to DEBUG and its only appender to A.
-log4j.logger.org.apache.pig=info, A
-
-# ***** A is set to be a ConsoleAppender.
-log4j.appender.A=org.apache.log4j.ConsoleAppender
-# ***** A uses PatternLayout.
-log4j.appender.A.layout=org.apache.log4j.PatternLayout
-log4j.appender.A.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/YARN/configuration/yarn-log4j.xml

@@ -0,0 +1,97 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+
+  <property>
+    <name>hadoop.mapreduce.jobsummary.logger</name>
+    <value>${hadoop.root.logger}</value>
+  </property>
+  <property>
+    <name>hadoop.mapreduce.jobsummary.log.file</name>
+    <value>hadoop-mapreduce.jobsummary.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.JSA</name>
+    <value>org.apache.log4j.DailyRollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.JSA.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.JSA.layout.ConversionPattern</name>
+    <value>%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.JSA.DatePattern</name>
+    <value>.yyyy-MM-dd</value>
+  </property>
+  <property>
+    <name>log4j.appender.JSA.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+
+  <!--Only for HDP2.0 below-->
+  <property>
+    <name>yarn.server.resourcemanager.appsummary.log.file</name>
+    <value>hadoop-mapreduce.jobsummary.log</value>
+  </property>
+  <property>
+    <name>yarn.server.resourcemanager.appsummary.logger</name>
+    <value>${hadoop.root.logger}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RMSUMMARY</name>
+    <value>org.apache.log4j.RollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.RMSUMMARY.File</name>
+    <value>/var/log/hadoop-yarn/yarn/${yarn.server.resourcemanager.appsummary.log.file}</value>
+  </property>
+  <property>
+    <name>log4j.appender.RMSUMMARY.MaxFileSize</name>
+    <value>256MB</value>
+  </property>
+  <property>
+    <name>log4j.appender.RMSUMMARY.MaxBackupIndex</name>
+    <value>20</value>
+  </property>
+  <property>
+    <name>log4j.appender.RMSUMMARY.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.RMSUMMARY.layout.ConversionPattern</name>
+    <value>%d{ISO8601} %p %c{2}: %m%n</value>
+  </property>
+  <property>
+    <name>log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary</name>
+    <value>${yarn.server.resourcemanager.appsummary.logger}</value>
+  </property>
+  <property>
+    <name>log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary</name>
+    <value>false</value>
+  </property>
+  <!--Only for HDP2.0 above-->
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/YARN/metainfo.xml

@@ -107,6 +107,7 @@
         <config-type>global</config-type>
         <config-type>mapred-site</config-type>
         <config-type>mapred-queue-acls</config-type>
+        <config-type>yarn-log4j</config-type>
       </configuration-dependencies>
     </service>
 
@@ -165,6 +166,7 @@
         <config-type>global</config-type>
         <config-type>mapred-site</config-type>
         <config-type>mapred-queue-acls</config-type>
+        <config-type>yarn-log4j</config-type>
       </configuration-dependencies>
     </service>
 

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/ZOOKEEPER/configuration/zookeeper-log4j.xml

@@ -0,0 +1,84 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+  <property>
+    <name>log4j.appender.CONSOLE</name>
+    <value>org.apache.log4j.ConsoleAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.CONSOLE.Threshold</name>
+    <value>INFO</value>
+  </property>
+  <property>
+    <name>log4j.appender.CONSOLE.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.CONSOLE.layout.ConversionPattern</name>
+    <value>%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.ROLLINGFILE</name>
+    <value>org.apache.log4j.RollingFileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.ROLLINGFILE.Threshold</name>
+    <value>DEBUG</value>
+  </property>
+  <property>
+    <name>log4j.appender.ROLLINGFILE.File</name>
+    <value>zookeeper.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.ROLLINGFILE.MaxFileSize</name>
+    <value>10MB</value>
+  </property>
+  <property>
+    <name>log4j.appender.ROLLINGFILE.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.ROLLINGFILE.layout.ConversionPattern</name>
+    <value>%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n</value>
+  </property>
+  <property>
+    <name>log4j.appender.TRACEFILE</name>
+    <value>org.apache.log4j.FileAppender</value>
+  </property>
+  <property>
+    <name>log4j.appender.TRACEFILE.Threshold</name>
+    <value>TRACE</value>
+  </property>
+  <property>
+    <name>log4j.appender.TRACEFILE.File</name>
+    <value>zookeeper_trace.log</value>
+  </property>
+  <property>
+    <name>log4j.appender.TRACEFILE.layout</name>
+    <value>org.apache.log4j.PatternLayout</value>
+  </property>
+  <property>
+    <name>log4j.appender.TRACEFILE.layout.ConversionPattern</name>
+    <value>%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n</value>
+  </property>
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/ZOOKEEPER/metainfo.xml

@@ -64,6 +64,7 @@
 
       <configuration-dependencies>
         <config-type>global</config-type>
+        <config-type>zookeeper-log4j</config-type>
       </configuration-dependencies>
     </service>
   </services>

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/ZOOKEEPER/package/scripts/params.py

@@ -69,3 +69,9 @@ security_enabled = config['configurations']['global']['security_enabled']
 smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
 smokeuser = config['configurations']['global']['smokeuser']
 kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+
+#log4j.properties
+if ('zookeeper-log4j' in config['configurations']):
+  log4j_props = config['configurations']['zookeeper-log4j']
+else:
+  log4j_props = None

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/ZOOKEEPER/package/scripts/zookeeper.py

@@ -19,6 +19,7 @@ limitations under the License.
 Ambari Agent
 
 """
+import os
 
 from resource_management import *
 import sys
@@ -63,7 +64,20 @@ def zookeeper(type = None):
          content = myid
     )
 
-  configFile("log4j.properties", template_name="log4j.properties.j2")
+  if (params.log4j_props != None):
+    PropertiesFile('log4j.properties',
+                   dir=params.config_dir,
+                   properties=params.log4j_props,
+                   mode=0664,
+                   owner=params.zk_user,
+                   group=params.user_group,
+    )
+  elif (os.path.exists(format("{params.config_dir}/log4j.properties"))):
+    File(format("{params.config_dir}/log4j.properties"),
+         mode=0644,
+         group=params.user_group,
+         owner=params.zk_user
+    )
 
   if params.security_enabled:
     if type == "server":

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/ZOOKEEPER/package/templates/log4j.properties.j2

@@ -1,71 +0,0 @@
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-#
-
-#
-# ZooKeeper Logging Configuration
-#
-
-# Format is "<default threshold> (, <appender>)+
-
-# DEFAULT: console appender only
-log4j.rootLogger=INFO, CONSOLE
-
-# Example with rolling log file
-#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE
-
-# Example with rolling log file and tracing
-#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE
-
-#
-# Log INFO level and above messages to the console
-#
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.Threshold=INFO
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n
-
-#
-# Add ROLLINGFILE to rootLogger to get log file output
-#    Log DEBUG level and above messages to a log file
-log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender
-log4j.appender.ROLLINGFILE.Threshold=DEBUG
-log4j.appender.ROLLINGFILE.File=zookeeper.log
-
-# Max log file size of 10MB
-log4j.appender.ROLLINGFILE.MaxFileSize=10MB
-# uncomment the next line to limit number of backup files
-#log4j.appender.ROLLINGFILE.MaxBackupIndex=10
-
-log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout
-log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n
-
-
-#
-# Add TRACEFILE to rootLogger to get log file output
-#    Log DEBUG level and above messages to a log file
-log4j.appender.TRACEFILE=org.apache.log4j.FileAppender
-log4j.appender.TRACEFILE.Threshold=TRACE
-log4j.appender.TRACEFILE.File=zookeeper_trace.log
-
-log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout
-### Notice we are including log4j's NDC here (%x)
-log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n