AMBARI-5900 Installation error on 2000 node cluster "ulimit" (dsen)

ambari-server/conf/unix/ambari.properties

@@ -46,3 +46,6 @@ agent.task.timeout=600
 # thread pool maximums
 client.threadpool.size.max=25
 agent.threadpool.size.max=25
+
+# linux open-file limit
+ulimit.open.files=10000

ambari-server/src/main/python/ambari-server.py

@@ -401,6 +401,10 @@ DEFAULT_DB_NAME = "ambari"
 STACK_LOCATION_KEY = 'metadata.path'
 STACK_LOCATION_DEFAULT = '/var/lib/ambari-server/resources/stacks'
 
+# linux open-file limit
+ULIMIT_OPEN_FILES_KEY = 'ulimit.open.files'
+ULIMIT_OPEN_FILES_DEFAULT = 10000
+
 #Apache License Header
 ASF_LICENSE_HEADER = '''
 # Copyright 2011 The Apache Software Foundation
@@ -2542,6 +2546,7 @@ def start(args):
 
   pidfile = PID_DIR + os.sep + PID_NAME
   command_base = SERVER_START_CMD_DEBUG if (SERVER_DEBUG_MODE or SERVER_START_DEBUG) else SERVER_START_CMD
+  command_base = "ulimit -n " + str(get_ulimit_open_files()) + "; " + command_base
   command = command_base.format(jdk_path, conf_dir, get_ambari_classpath(), pidfile)
   if not os.path.exists(PID_DIR):
     os.makedirs(PID_DIR, 0755)
@@ -4021,6 +4026,19 @@ def get_fqdn():
     return socket.getfqdn()
 
 
+def get_ulimit_open_files():
+  properties = get_ambari_properties()
+  if properties == -1:
+    print "Error reading ambari properties"
+    return None
+
+  open_files = int(properties[ULIMIT_OPEN_FILES_KEY])
+  if open_files > 0:
+    return open_files
+  else:
+    return ULIMIT_OPEN_FILES_DEFAULT
+
+
 def is_valid_filepath(filepath):
   if not filepath or not os.path.exists(filepath) or os.path.isdir(filepath):
     print 'Invalid path, please provide the absolute file path.'

ambari-server/src/test/python/TestAmbariServer.py

@@ -1683,6 +1683,35 @@ class TestAmbariServer(TestCase):
     self.assertEqual(fqdn, host)
 
 
+  @patch.object(ambari_server, "find_properties_file")
+  def test_get_ulimit_open_files(self, find_properties_file_mock):
+
+    # 1 - No ambari.properties
+    find_properties_file_mock.return_value = None
+    open_files = ambari_server.get_fqdn()
+    self.assertEqual(open_files, None)
+
+    # 2 - With ambari.properties - ok
+    tf1 = tempfile.NamedTemporaryFile()
+    prop_value = 65000
+    with open(tf1.name, 'w') as fout:
+      fout.write(ambari_server.ULIMIT_OPEN_FILES_KEY + '=' + str(prop_value))
+    fout.close()
+    find_properties_file_mock.return_value = tf1.name
+    open_files = ambari_server.get_ulimit_open_files()
+    self.assertEqual(open_files, 65000)
+
+    # 2 - With ambari.properties - default
+    tf1 = tempfile.NamedTemporaryFile()
+    prop_value = 0
+    with open(tf1.name, 'w') as fout:
+      fout.write(ambari_server.ULIMIT_OPEN_FILES_KEY + '=' + str(prop_value))
+    fout.close()
+    find_properties_file_mock.return_value = tf1.name
+    open_files = ambari_server.get_ulimit_open_files()
+    self.assertEqual(open_files, ambari_server.ULIMIT_OPEN_FILES_DEFAULT)
+
+
   @patch.object(ambari_server, "run_os_command")
   def test_get_cert_info(self, run_os_command_mock):
     # Error running openssl command