AMBARI-11960: NonRoot: Address ambari-server and ambari-agent non-root issues (jluniya)

ambari-common/src/main/python/ambari_commons/os_linux.py

@@ -29,6 +29,7 @@ from ambari_commons.logging_utils import print_info_msg, print_warning_msg
 
 NR_CHMOD_CMD = 'chmod {0} {1} {2}'
 NR_CHOWN_CMD = 'chown {0} {1} {2}'
+WARN_MSG = "Command {0} returned exit code {1} with message: {2}"
 
 ULIMIT_CMD = "ulimit -n"
 
@@ -48,10 +49,15 @@ def os_run_os_command(cmd, env=None, shell=False, cwd=None):
   (stdoutdata, stderrdata) = process.communicate()
   return process.returncode, stdoutdata, stderrdata
 
-def os_change_owner(filePath, user):
-  uid = pwd.getpwnam(user).pw_uid
-  gid = pwd.getpwnam(user).pw_gid
-  os.chown(filePath, uid, gid)
+def os_change_owner(filePath, user, recursive):
+  if recursive:
+    params = " -R "
+  else:
+    params = ""
+  command = NR_CHOWN_CMD.format(params, user, filePath)
+  retcode, out, err = os_run_os_command(command)
+  if retcode != 0:
+    print_warning_msg(WARN_MSG.format(command, filePath, err))
 
 def os_is_root():
   '''
@@ -61,7 +67,6 @@ def os_is_root():
   return os.geteuid() == 0
 
 def os_set_file_permissions(file, mod, recursive, user):
-  WARN_MSG = "Command {0} returned exit code {1} with message: {2}"
   if recursive:
     params = " -R "
   else:
@@ -70,15 +75,11 @@ def os_set_file_permissions(file, mod, recursive, user):
   retcode, out, err = os_run_os_command(command)
   if retcode != 0:
     print_warning_msg(WARN_MSG.format(command, file, err))
-  command = NR_CHOWN_CMD.format(params, user, file)
-  retcode, out, err = os_run_os_command(command)
-  if retcode != 0:
-    print_warning_msg(WARN_MSG.format(command, file, err))
+  os_change_owner(file, user, recursive)
 
 def os_set_open_files_limit(maxOpenFiles):
   command = "%s %s" % (ULIMIT_CMD, str(maxOpenFiles))
   os_run_os_command(command)
 
-
 def os_getpass(prompt):
   return getpass.unix_getpass(prompt)

ambari-common/src/main/python/ambari_commons/os_utils.py

@@ -109,8 +109,8 @@ def is_root():
   return os_is_root()
 
 # Proxy to the os implementation
-def change_owner(filePath, user):
-  os_change_owner(filePath, user)
+def change_owner(filePath, user, recursive):
+  os_change_owner(filePath, user, recursive)
 
 # Proxy to the os implementation
 def set_open_files_limit(maxOpenFiles):

ambari-common/src/main/python/ambari_commons/os_windows.py

@@ -364,8 +364,10 @@ def run_powershell_script(script_content):
   os.remove(script_file.name)
   return result
 
-def os_change_owner(filePath, user):
+def os_change_owner(filePath, user, recursive):
   cmd = ['icacls', filePath, '/setowner', user]
+  if recursive:
+    cmd = ['icacls', filePath, '/t', '/setowner', user]
   retcode, outdata, errdata = os_run_os_command(cmd)
   return retcode
 

ambari-server/src/main/python/ambari_server/serverConfiguration.py

@@ -189,6 +189,7 @@ class ServerConfigDefaults(object):
     # Rules are executed in the same order as they are listed
     # {0} in user/group will be replaced by customized ambari-server username
     self.NR_ADJUST_OWNERSHIP_LIST = []
+    self.NR_CHANGE_OWNERSHIP_LIST = []
     self.NR_USERADD_CMD = ""
 
     self.MASTER_KEY_FILE_PERMISSIONS = "600"
@@ -297,6 +298,7 @@ class ServerConfigDefaultsLinux(ServerConfigDefaults):
       ("/var/lib/ambari-server/keys/db", "700", "{0}", False),
       ("/var/lib/ambari-server/keys/db/newcerts", "700", "{0}", False),
       ("/var/lib/ambari-server/keys/.ssh", "700", "{0}", False),
+      ("/var/lib/ambari-server/resources/common-services/", "755", "{0}", True),
       ("/var/lib/ambari-server/resources/stacks/", "755", "{0}", True),
       ("/var/lib/ambari-server/resources/custom_actions/", "755", "{0}", True),
       ("/var/lib/ambari-server/resources/host_scripts/", "755", "{0}", True),
@@ -310,6 +312,8 @@ class ServerConfigDefaultsLinux(ServerConfigDefaults):
       ("/etc/ambari-server/conf/ldap-password.dat", "640", "{0}", False),
       ("/var/run/ambari-server/stack-recommendations/", "744", "{0}", True),
       ("/var/run/ambari-server/stack-recommendations/", "755", "{0}", False),
+      ("/var/lib/ambari-server/resources/data/", "644", "{0}", False),
+      ("/var/lib/ambari-server/resources/data/", "755", "{0}", False),
       ("/var/lib/ambari-server/data/tmp/", "644", "{0}", True),
       ("/var/lib/ambari-server/data/tmp/", "755", "{0}", False),
       ("/var/lib/ambari-server/data/cache/", "600", "{0}", True),
@@ -317,6 +321,13 @@ class ServerConfigDefaultsLinux(ServerConfigDefaults):
       # Also, /etc/ambari-server/conf/password.dat
       # is generated later at store_password_file
     ]
+    self.NR_CHANGE_OWNERSHIP_LIST = [
+      ("/var/lib/ambari-server", "{0}", True),
+      ("/usr/lib/ambari-server", "{0}", True),
+      ("/var/log/ambari-server", "{0}", True),
+      ("/var/run/ambari-server", "{0}", True),
+      ("/etc/ambari-server", "{0}", True),
+    ]
     self.NR_USERADD_CMD = 'useradd -M --comment "{1}" ' \
                  '--shell %s -d /var/lib/ambari-server/keys/ {0}' % locate_file('nologin', '/sbin')
 

ambari-server/src/main/python/ambari_server/setupSecurity.py

@@ -32,7 +32,7 @@ from ambari_commons.logging_utils import print_warning_msg, print_error_msg, pri
 from ambari_commons.os_check import OSConst
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 from ambari_commons.os_utils import is_root, set_file_permissions, \
-  run_os_command, search_file, is_valid_filepath
+  run_os_command, search_file, is_valid_filepath, change_owner
 from ambari_server.serverConfiguration import configDefaults, \
   encrypt_password, find_jdk, find_properties_file, get_alias_string, get_ambari_properties, get_conf_dir, \
   get_credential_store_location, get_full_ambari_classpath, get_is_persisted, get_is_secure, get_master_key_location, \
@@ -44,7 +44,8 @@ from ambari_server.serverConfiguration import configDefaults, \
   LDAP_PRIMARY_URL_PROPERTY, SECURITY_IS_ENCRYPTION_ENABLED, SECURITY_KEY_ENV_VAR_NAME, SECURITY_KERBEROS_JASS_FILENAME, \
   SECURITY_PROVIDER_KEY_CMD, SECURITY_MASTER_KEY_FILENAME, SSL_TRUSTSTORE_PASSWORD_ALIAS, \
   SSL_TRUSTSTORE_PASSWORD_PROPERTY, SSL_TRUSTSTORE_PATH_PROPERTY, SSL_TRUSTSTORE_TYPE_PROPERTY, \
-  SSL_API, SSL_API_PORT, DEFAULT_SSL_API_PORT, CLIENT_API_PORT
+  SSL_API, SSL_API_PORT, DEFAULT_SSL_API_PORT, CLIENT_API_PORT, JDK_NAME_PROPERTY, JCE_NAME_PROPERTY, JAVA_HOME_PROPERTY, \
+  get_resources_location
 from ambari_server.serverUtils import is_server_runing, get_ambari_server_api_base
 from ambari_server.setupActions import SETUP_ACTION, LDAP_SETUP_ACTION
 from ambari_server.userInput import get_validated_string_input, get_prompt_default, read_password, get_YN_input
@@ -134,13 +135,33 @@ def adjust_directory_permissions(ambari_user):
   keyLocation = get_master_key_location(properties)
   masterKeyFile = search_file(SECURITY_MASTER_KEY_FILENAME, keyLocation)
   if masterKeyFile:
-    configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((masterKeyFile, configDefaults.MASTER_KEY_FILE_PERMISSIONS, "{0}", "{0}", False))
+    configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((masterKeyFile, configDefaults.MASTER_KEY_FILE_PERMISSIONS, "{0}", False))
   credStoreFile = get_credential_store_location(properties)
   if os.path.exists(credStoreFile):
-    configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((credStoreFile, configDefaults.CREDENTIALS_STORE_FILE_PERMISSIONS, "{0}", "{0}", False))
+    configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((credStoreFile, configDefaults.CREDENTIALS_STORE_FILE_PERMISSIONS, "{0}", False))
   trust_store_location = properties[SSL_TRUSTSTORE_PATH_PROPERTY]
   if trust_store_location:
-    configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((trust_store_location, configDefaults.TRUST_STORE_LOCATION_PERMISSIONS, "{0}", "{0}", False))
+    configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((trust_store_location, configDefaults.TRUST_STORE_LOCATION_PERMISSIONS, "{0}", False))
+
+  # Update JDK and JCE permissions
+  resources_dir = get_resources_location(properties)
+  jdk_file_name = properties.get_property(JDK_NAME_PROPERTY)
+  jce_file_name = properties.get_property(JCE_NAME_PROPERTY)
+  java_home = properties.get_property(JAVA_HOME_PROPERTY)
+  if jdk_file_name:
+    jdk_file_path = os.path.abspath(os.path.join(resources_dir, jdk_file_name))
+    if(os.path.exists(jdk_file_path)):
+      configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((jdk_file_path, "644", "{0}", False))
+  if jce_file_name:
+    jce_file_path = os.path.abspath(os.path.join(resources_dir, jce_file_name))
+    if(os.path.exists(jce_file_path)):
+      configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((jce_file_path, "644", "{0}", False))
+  if java_home:
+    jdk_security_dir = os.path.abspath(os.path.join(java_home, configDefaults.JDK_SECURITY_DIR))
+    if(os.path.exists(jce_file_path)):
+      configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((jdk_security_dir, "644", "{0}", True))
+      configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((jdk_security_dir, "755", "{0}", False))
+
   print "Adjusting ambari-server permissions and ownership..."
 
   for pack in configDefaults.NR_ADJUST_OWNERSHIP_LIST:
@@ -148,8 +169,16 @@ def adjust_directory_permissions(ambari_user):
     mod = pack[1]
     user = pack[2].format(ambari_user)
     recursive = pack[3]
+    print_info_msg("Setting file permissions: {0} {1} {2} {3}".format(file, mod, user, recursive))
     set_file_permissions(file, mod, user, recursive)
 
+  for pack in configDefaults.NR_CHANGE_OWNERSHIP_LIST:
+    path = pack[0]
+    user = pack[1].format(ambari_user)
+    recursive = pack[2]
+    print_info_msg("Changing ownership: {0} {1} {2}".format(path, user, recursive))
+    change_owner(path, user, recursive)
+
 def configure_ldap_password():
   passwordDefault = ""
   passwordPrompt = 'Enter Manager Password* : '

ambari-server/src/main/resources/custom_actions/scripts/check_host.py

@@ -23,6 +23,7 @@ Ambari Agent
 import os
 import subprocess
 import socket
+import getpass
 
 from resource_management.libraries.functions import packages_analyzer
 from ambari_commons import os_utils
@@ -31,6 +32,8 @@ from ambari_commons.inet_utils import download_file
 from resource_management import Script, Execute, format
 from ambari_agent.HostInfo import HostInfo
 from ambari_agent.HostCheckReportFileHandler import HostCheckReportFileHandler
+from resource_management.core.resources import Directory, File
+from ambari_commons.constants import AMBARI_SUDO_BINARY
 
 CHECK_JAVA_HOME = "java_home_check"
 CHECK_DB_CONNECTION = "db_connection_check"
@@ -170,14 +173,14 @@ class CheckHost(Script):
 
   def execute_java_home_available_check(self, config):
     print "Java home check started."
-    java64_home = config['commandParams']['java_home']
+    java_home = config['commandParams']['java_home']
 
-    print "Java home to check: " + java64_home
+    print "Java home to check: " + java_home
     java_bin = "java"
     if OSCheck.is_windows_family():
       java_bin = "java.exe"
   
-    if not os.path.isfile(os.path.join(java64_home, "bin", java_bin)):
+    if not os.path.isfile(os.path.join(java_home, "bin", java_bin)):
       print "Java home doesn't exist!"
       java_home_check_structured_output = {"exit_code" : 1, "message": "Java home doesn't exist!"}
     else:
@@ -195,7 +198,7 @@ class CheckHost(Script):
     ambari_server_hostname = config['commandParams']['ambari_server_host']
     check_db_connection_jar_name = "DBConnectionVerification.jar"
     jdk_location = config['commandParams']['jdk_location']
-    java64_home = config['commandParams']['java_home']
+    java_home = config['commandParams']['java_home']
     db_name = config['commandParams']['db_name']
 
     if db_name == DB_MYSQL:
@@ -229,7 +232,7 @@ class CheckHost(Script):
       java_bin = "java.exe"
       class_path_delimiter = ";"
 
-    java_exec = os.path.join(java64_home, "bin",java_bin)
+    java_exec = os.path.join(java_home, "bin",java_bin)
 
     if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \
         or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec):
@@ -245,7 +248,7 @@ class CheckHost(Script):
       jdk_name = config['commandParams']['jdk_name']
       jdk_url = "{0}/{1}".format(jdk_location, jdk_name)
       jdk_download_target = os.path.join(agent_cache_dir, jdk_name)
-      java_dir = os.path.dirname(java64_home)
+      java_dir = os.path.dirname(java_home)
       try:
         download_file(jdk_url, jdk_download_target)
       except Exception, e:
@@ -255,26 +258,38 @@ class CheckHost(Script):
         db_connection_check_structured_output = {"exit_code" : 1, "message": message}
         return db_connection_check_structured_output
 
-      if jdk_name.endswith(".bin"):
-        install_cmd = format("mkdir -p {java_dir} ; chmod +x {jdk_download_target}; cd {java_dir} ; echo A | " \
-                           "{jdk_curl_target} -noregister > /dev/null 2>&1")
-        install_path = ["/bin","/usr/bin/"]
-      elif jdk_name.endswith(".gz"):
-        install_cmd = format("mkdir -p {java_dir} ; cd {java_dir} ; tar -xf {jdk_download_target} > /dev/null 2>&1")
-        install_path = ["/bin","/usr/bin/"]
-      elif jdk_name.endswith(".exe"):
+      if jdk_name.endswith(".exe"):
         install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format(
-          os_utils.quote_path(jdk_download_target), os_utils.quote_path(java64_home),
+        os_utils.quote_path(jdk_download_target), os_utils.quote_path(java_home),
         )
         install_path = [java_dir]
-
-      try:
-        Execute(install_cmd, path = install_path)
-      except Exception, e:
-        message = "Error installing java.\n" + str(e)
-        print message
-        db_connection_check_structured_output = {"exit_code" : 1, "message": message}
-        return db_connection_check_structured_output
+        try:
+          Execute(install_cmd, path = install_path)
+        except Exception, e:
+          message = "Error installing java.\n" + str(e)
+          print message
+          db_connection_check_structured_output = {"exit_code" : 1, "message": message}
+          return db_connection_check_structured_output
+      else:
+        tmp_java_dir = format("{tmp_dir}/jdk")
+        sudo = AMBARI_SUDO_BINARY
+        if jdk_name.endswith(".bin"):
+          chmod_cmd = ("chmod", "+x", jdk_download_target)
+          install_cmd = format("mkdir -p {tmp_java_dir} && cd {tmp_java_dir} && echo A | {jdk_download_target} -noregister && {sudo} cp -rp {tmp_java_dir}/* {java_dir}")
+        elif jdk_name.endswith(".gz"):
+          chmod_cmd = ("chmod","a+x", java_dir)
+          install_cmd = format("mkdir -p {tmp_java_dir} && cd {tmp_java_dir} && tar -xf {jdk_download_target} && {sudo} cp -rp {tmp_java_dir}/* {java_dir}")
+        try:
+          Directory(java_dir)
+          Execute(chmod_cmd, not_if = format("test -e {java_exec}"), sudo = True)
+          Execute(install_cmd, not_if = format("test -e {java_exec}"))
+          File(format("{java_home}/bin/java"), mode=0755, cd_access="a")
+          Execute(("chown","-R", getpass.getuser(), java_home), sudo = True)
+        except Exception, e:
+          message = "Error installing java.\n" + str(e)
+          print message
+          db_connection_check_structured_output = {"exit_code" : 1, "message": message}
+          return db_connection_check_structured_output
 
     # download DBConnectionVerification.jar from ambari-server resources
     try:

ambari-server/src/test/python/TestAmbariServer.py

@@ -1004,37 +1004,50 @@ class TestAmbariServer(TestCase):
   @patch("os.path.exists")
   @patch("ambari_server.setupSecurity.set_file_permissions")
   @patch("ambari_server.setupSecurity.get_ambari_properties")
+  @patch("ambari_server.setupSecurity.get_resources_location")
   @patch("ambari_server.setupSecurity.get_value_from_properties")
   @patch("os.mkdir")
   @patch("shutil.rmtree")
-  def test_adjust_directory_permissions(self, rmtree_mock, mkdir_mock,
-                                        get_value_from_properties_mock, get_ambari_properties_mock,
-                                        set_file_permissions_mock, exists_mock):
+  @patch("ambari_commons.os_utils.print_info_msg")
+  @patch("ambari_server.setupSecurity.change_owner")
+  def test_adjust_directory_permissions(self, change_owner_mock, print_info_msg_mock, rmtree_mock, mkdir_mock,
+                                        get_value_from_properties_mock, get_resources_location_mock,
+                                        get_ambari_properties_mock, set_file_permissions_mock, exists_mock):
     # Testing boostrap dir wipe
     properties_mock = MagicMock()
     get_value_from_properties_mock.return_value = "dummy_bootstrap_dir"
+    get_resources_location_mock.return_value = "dummy_resources_dir"
     exists_mock.return_value = False
     adjust_directory_permissions("user")
     self.assertEquals(rmtree_mock.call_args_list[0][0][0], os.path.join(os.getcwd(), "dummy_bootstrap_dir"))
     self.assertTrue(mkdir_mock.called)
 
     set_file_permissions_mock.reset_mock()
+    change_owner_mock.reset_mock()
     # Test recursive calls
-    old_list = configDefaults.NR_ADJUST_OWNERSHIP_LIST
-
+    old_adjust_owner_list = configDefaults.NR_ADJUST_OWNERSHIP_LIST
+    old_change_owner_list = configDefaults.NR_CHANGE_OWNERSHIP_LIST
     try:
       configDefaults.NR_ADJUST_OWNERSHIP_LIST = [
         ( "/etc/ambari-server/conf", "755", "{0}", True ),
         ( "/etc/ambari-server/conf/ambari.properties", "644", "{0}", False )
       ]
 
+      configDefaults.NR_CHANGE_OWNERSHIP_LIST = [
+        ( "/etc/ambari-server", "{0}", True )
+      ]
+
       adjust_directory_permissions("user")
       self.assertTrue(len(set_file_permissions_mock.call_args_list) ==
                       len(configDefaults.NR_ADJUST_OWNERSHIP_LIST))
       self.assertEquals(set_file_permissions_mock.call_args_list[0][0][3], True)
       self.assertEquals(set_file_permissions_mock.call_args_list[1][0][3], False)
+      self.assertTrue(len(change_owner_mock.call_args_list) ==
+                      len(configDefaults.NR_CHANGE_OWNERSHIP_LIST))
+      self.assertEquals(change_owner_mock.call_args_list[0][0][2], True)
     finally:
-      configDefaults.NR_ADJUST_OWNERSHIP_LIST = old_list
+      configDefaults.NR_ADJUST_OWNERSHIP_LIST = old_adjust_owner_list
+      configDefaults.NR_CHANGE_OWNERSHIP_LIST = old_change_owner_list
     pass
 
   @not_for_platform(PLATFORM_WINDOWS)