AMBARI-13290. Set kafka userid ulimit open files to 32k for kafka broker through Ambari (aonishuk)

ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-env.xml

@@ -42,6 +42,16 @@
       <overridable>false</overridable>
     </value-attributes>
   </property>
+  <property>
+    <name>kafka_user_nofile_limit</name>
+    <value>32000</value>
+    <description>Max open files limit setting for KAFKA user.</description>
+  </property>
+  <property>
+    <name>kafka_user_nproc_limit</name>
+    <value>16000</value>
+    <description>Max number of processes limit setting for KAFKA user.</description>
+  </property>
 
   <!-- kafka-env.sh -->
   <property>

ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py

@@ -94,6 +94,19 @@ def kafka():
         TemplateConfig(format("{conf_dir}/kafka_client_jaas.conf"),
                        owner=params.kafka_user)
 
+    # On some OS this folder could be not exists, so we will create it before pushing there files
+    Directory(params.limits_conf_dir,
+              recursive=True,
+              owner='root',
+              group='root'
+    )
+
+    File(os.path.join(params.limits_conf_dir, 'kafka.conf'),
+         owner='root',
+         group='root',
+         mode=0644,
+         content=Template("kafka.conf.j2")
+    )
 
     setup_symlink(params.kafka_managed_pid_dir, params.kafka_pid_dir)
     setup_symlink(params.kafka_managed_log_dir, params.kafka_log_dir)

ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py

@@ -43,6 +43,10 @@ hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
 kafka_home = '/usr/lib/kafka/'
 kafka_bin = kafka_home+'/bin/kafka'
 conf_dir = "/etc/kafka/conf"
+limits_conf_dir = "/etc/security/limits.d"
+
+kafka_user_nofile_limit = config['configurations']['kafka-env']['kafka_user_nofile_limit']
+kafka_user_nproc_limit = config['configurations']['kafka-env']['kafka_user_nproc_limit']
 
 # parameters for 2.2+
 if Script.is_hdp_stack_greater_or_equal("2.2"):

ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/templates/kafka.conf.j2

@@ -0,0 +1,35 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{kafka_user}}   - nofile   {{kafka_user_nofile_limit}}
+{{kafka_user}}   - nproc    {{kafka_user_nproc_limit}}