AMBARI-15681. Ranger Admin/Usersync: Make log4j configurable from Ambari(gautam)

ambari-common/src/main/python/resource_management/libraries/functions/constants.py

@@ -77,4 +77,5 @@ class StackFeature:
   HIVE_PURGE_TABLE = "hive_purge_table"
   HIVE_SERVER2_KERBERIZED_ENV = "hive_server2_kerberized_env"
   HIVE_ENV_HEAPSIZE = "hive_env_heapsize"
-  RANGER_KMS_HSM_SUPPORT = "ranger_kms_hsm_support"
+  RANGER_KMS_HSM_SUPPORT = "ranger_kms_hsm_support"
+  RANGER_LOG4J_SUPPORT = "ranger_log4j_support"

ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py

@@ -223,6 +223,11 @@ _DEFAULT_STACK_FEATURES = {
       "name": "ranger_kms_hsm_support",
       "description": "Ranger KMS HSM support (AMBARI-15752)",
       "min_version": "2.5.0.0"
+    },
+    {
+      "name": "ranger_log4j_support",
+      "description": "Ranger supporting log-4j properties (AMBARI-15681)",
+      "min_version": "2.5.0.0"
     }
   ]
 }

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py

@@ -60,6 +60,7 @@ stack_supports_config_versioning =  stack_version_formatted and check_stack_feat
 stack_supports_usersync_non_root =  stack_version_formatted and check_stack_feature(StackFeature.RANGER_USERSYNC_NON_ROOT, stack_version_formatted)
 stack_supports_ranger_tagsync =  stack_version_formatted and check_stack_feature(StackFeature.RANGER_TAGSYNC_COMPONENT, stack_version_formatted)
 stack_supports_ranger_audit_db = stack_version_formatted and check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, stack_version_formatted)
+stack_supports_ranger_log4j =  stack_version_formatted and check_stack_feature(StackFeature.RANGER_LOG4J_SUPPORT, stack_version_formatted)
 
 downgrade_from_version = default("/commandParams/downgrade_from_version", None)
 upgrade_direction = default("/commandParams/upgrade_direction", None)
@@ -110,6 +111,8 @@ ranger_admin_default_file = format('{ranger_conf}/ranger-admin-default-site.xml'
 security_app_context_file = format('{ranger_conf}/security-applicationContext.xml')
 ranger_ugsync_default_file = format('{ranger_ugsync_conf}/ranger-ugsync-default.xml')
 usgsync_log4j_file = format('{ranger_ugsync_conf}/log4j.xml')
+if stack_supports_ranger_log4j:
+  usgsync_log4j_file = format('{ranger_ugsync_conf}/log4j.properties')
 cred_validator_file = format('{usersync_home}/native/credValidator.uexe')
 
 ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
@@ -225,3 +228,8 @@ ranger_tagsync_tagadmin_password = unicode(config["configurations"]["ranger-tags
 tagsync_jceks_path = config["configurations"]["ranger-tagsync-site"]["ranger.tagsync.tagadmin.keystore"]
 tagsync_application_properties = dict(config["configurations"]["tagsync-application-properties"]) if has_ranger_tagsync else None
 tagsync_pid_file = format('{ranger_pid_dir}/tagsync.pid')
+
+# ranger log4j.properties
+admin_log4j = config['configurations']['admin-log4j']['content']
+usersync_log4j = config['configurations']['usersync-log4j']['content']
+tagsync_log4j = config['configurations']['tagsync-log4j']['content']

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py

@@ -163,6 +163,14 @@ def setup_ranger_admin(upgrade_type=None):
     group=params.unix_group,
   )
 
+  if params.stack_supports_ranger_log4j:
+    File(format('{ranger_home}/ews/webapp/WEB-INF/log4j.properties'),
+      owner=params.unix_user,
+      group=params.unix_group,
+      content=params.admin_log4j,
+      mode=0644
+    )
+
   do_keystore_setup(upgrade_type=upgrade_type)
 
 
@@ -348,6 +356,14 @@ def setup_usersync(upgrade_type=None):
     dst_file = format('{usersync_home}/conf/ranger-ugsync-default.xml')
     Execute(('cp', '-f', src_file, dst_file), sudo=True)
 
+  if params.stack_supports_ranger_log4j:
+    File(format('{usersync_home}/conf/log4j.properties'),
+      owner=params.unix_user,
+      group=params.unix_group,
+      content=params.usersync_log4j,
+      mode=0644
+    )
+  elif upgrade_type is not None and not params.stack_supports_ranger_log4j:
     src_file = format('{usersync_home}/conf.dist/log4j.xml')
     dst_file = format('{usersync_home}/conf/log4j.xml')
     Execute(('cp', '-f', src_file, dst_file), sudo=True)
@@ -462,7 +478,13 @@ def setup_tagsync(upgrade_type=None):
     group=params.unix_group
   )
 
-  if upgrade_type is not None:
+  if params.stack_supports_ranger_log4j:
+    File(format('{ranger_tagsync_conf}/log4j.properties'),
+      owner=params.unix_user,
+      group=params.unix_group,
+      content=params.tagsync_log4j,
+      mode=0644
+    )
     src_file = format('{ranger_tagsync_home}/conf.dist/log4j.xml')
     dst_file = format('{tagsync_log4j_file}')
     Execute(('cp', '-f', src_file, dst_file), sudo=True)

ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json

@@ -200,6 +200,11 @@
       "name": "ranger_kms_hsm_support",
       "description": "Ranger KMS HSM support (AMBARI-15752)",
       "min_version": "2.5.0.0"
+    },
+    {
+      "name": "ranger_log4j_support",
+      "description": "Ranger supporting log-4j properties (AMBARI-15681)",
+      "min_version": "2.5.0.0"
     }
   ]
 }

ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/admin-log4j.xml

@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+
+    <property>
+        <name>content</name>
+        <description>admin-log4j.properties</description>
+        <value>
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+log4j.rootLogger = warn,xa_log_appender
+
+
+# xa_logger
+log4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.xa_log_appender.file=${logdir}/xa_portal.log
+log4j.appender.xa_log_appender.datePattern='.'yyyy-MM-dd
+log4j.appender.xa_log_appender.append=true
+log4j.appender.xa_log_appender.layout=org.apache.log4j.PatternLayout
+log4j.appender.xa_log_appender.layout.ConversionPattern=%d [%t] %-5p %C{6} (%F:%L) - %m%n
+# xa_log_appender : category and additivity
+log4j.category.org.springframework=warn,xa_log_appender
+log4j.additivity.org.springframework=false
+
+log4j.category.org.apache.ranger=info,xa_log_appender
+log4j.additivity.org.apache.ranger=false
+
+log4j.category.xa=info,xa_log_appender
+log4j.additivity.xa=false
+
+# perf_logger
+log4j.appender.perf_appender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.perf_appender.file=${logdir}/ranger_admin_perf.log
+log4j.appender.perf_appender.datePattern='.'yyyy-MM-dd
+log4j.appender.perf_appender.append=true
+log4j.appender.perf_appender.layout=org.apache.log4j.PatternLayout
+log4j.appender.perf_appender.layout.ConversionPattern=%d [%t] %m%n
+
+
+# sql_appender
+log4j.appender.sql_appender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.sql_appender.file=${logdir}/xa_portal_sql.log
+log4j.appender.sql_appender.datePattern='.'yyyy-MM-dd
+log4j.appender.sql_appender.append=true
+log4j.appender.sql_appender.layout=org.apache.log4j.PatternLayout
+log4j.appender.sql_appender.layout.ConversionPattern=%d [%t] %-5p %C{6} (%F:%L) - %m%n
+
+# sql_appender : category and additivity
+log4j.category.org.hibernate.SQL=warn,sql_appender
+log4j.additivity.org.hibernate.SQL=false
+
+log4j.category.jdbc.sqlonly=fatal,sql_appender
+log4j.additivity.jdbc.sqlonly=false
+
+log4j.category.jdbc.sqltiming=warn,sql_appender
+log4j.additivity.jdbc.sqltiming=false
+
+log4j.category.jdbc.audit=fatal,sql_appender
+log4j.additivity.jdbc.audit=false
+
+log4j.category.jdbc.resultset=fatal,sql_appender
+log4j.additivity.jdbc.resultset=false
+
+log4j.category.jdbc.connection=fatal,sql_appender
+log4j.additivity.jdbc.connection=false
+        </value>
+        <value-attributes>
+            <type>content</type>
+            <show-property-name>false</show-property-name>
+        </value-attributes>
+    </property>
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/tagsync-log4j.xml

@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+
+    <property>
+        <name>content</name>
+        <description>tagsync-log4j.properties</description>
+        <value>
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+log4j.rootLogger = info,logFile
+
+# logFile
+log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.logFile.file=${logdir}/tagsync.log
+log4j.appender.logFile.datePattern='.'yyyy-MM-dd
+log4j.appender.logFile.layout=org.apache.log4j.PatternLayout
+log4j.appender.logFile.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %L %m%n
+
+# console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.Target=System.out
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %L %m%n
+        </value>
+        <value-attributes>
+            <type>content</type>
+            <show-property-name>false</show-property-name>
+        </value-attributes>
+    </property>
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/usersync-log4j.xml

@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+
+    <property>
+        <name>content</name>
+        <description>usersync-log4j.properties</description>
+        <value>
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+log4j.rootLogger = info,logFile
+
+# logFile
+log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.logFile.file=${logdir}/usersync.log
+log4j.appender.logFile.datePattern='.'yyyy-MM-dd
+log4j.appender.logFile.layout=org.apache.log4j.PatternLayout
+log4j.appender.logFile.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n
+
+# console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.Target=System.out
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n
+        </value>
+        <value-attributes>
+            <type>content</type>
+            <show-property-name>false</show-property-name>
+        </value-attributes>
+    </property>
+
+</configuration>