AMBARI-9228. Ambari Server setup to install and copy JCE policy file in-place (handle both Default / Custom JDK scenarios) (rlevas)

ambari-server/sbin/ambari-server

@@ -111,6 +111,10 @@ case "$1" in
         echo -e "Setup ambari-server"
         $PYTHON /usr/sbin/ambari-server.py $@
         ;;
+  setup-jce)
+        echo -e "Updating jce policy"
+        $PYTHON /usr/sbin/ambari-server.py $@
+        ;;
   setup-ldap)
         echo -e "Setting up LDAP properties..."
         $PYTHON /usr/sbin/ambari-server.py $@
@@ -137,7 +141,7 @@ case "$1" in
         ;;
   *)
         echo "Usage: /usr/sbin/ambari-server
-        {start|stop|restart|setup|upgrade|status|upgradestack|setup-ldap|sync-ldap|setup-security|refresh-stack-hash|backup|restore} [options]
+        {start|stop|restart|setup|setup-jce|upgrade|status|upgradestack|setup-ldap|sync-ldap|setup-security|refresh-stack-hash|backup|restore} [options]
         Use usr/sbin/ambari-server <action> --help to get details on options available.
         Or, simply invoke ambari-server.py --help to print the options."
         exit 1

ambari-server/src/main/python/ambari-server.py

@@ -33,13 +33,13 @@ from ambari_server.BackupRestore import main as BackupRestore_main
 from ambari_server.dbConfiguration import DATABASE_NAMES
 from ambari_server.serverConfiguration import configDefaults, get_ambari_properties, PID_NAME
 from ambari_server.serverUtils import is_server_runing, refresh_stack_hash
-from ambari_server.serverSetup import reset, setup
+from ambari_server.serverSetup import reset, setup, setup_jce_policy
 from ambari_server.serverUpgrade import upgrade, upgrade_stack
 from ambari_server.setupHttps import setup_https, setup_ganglia_https
 
 from ambari_server.setupActions import BACKUP_ACTION, LDAP_SETUP_ACTION, LDAP_SYNC_ACTION, PSTART_ACTION, \
   REFRESH_STACK_HASH_ACTION, RESET_ACTION, RESTORE_ACTION, SETUP_ACTION, SETUP_SECURITY_ACTION, START_ACTION, \
-  STATUS_ACTION, STOP_ACTION, UPGRADE_ACTION, UPGRADE_STACK_ACTION
+  STATUS_ACTION, STOP_ACTION, UPGRADE_ACTION, UPGRADE_STACK_ACTION, SETUP_JCE_ACTION
 from ambari_server.setupSecurity import setup_ldap, sync_ldap, setup_master_key, setup_ambari_krb5_jaas
 from ambari_server.userInput import get_validated_string_input
 
@@ -491,6 +491,7 @@ def create_user_action_map(args, options):
 def create_user_action_map(args, options):
   action_map = {
         SETUP_ACTION: UserAction(setup, options),
+        SETUP_JCE_ACTION : UserActionPossibleArgs(setup_jce_policy, [2], args),
         START_ACTION: UserAction(start, options),
         STOP_ACTION: UserAction(stop, options),
         RESET_ACTION: UserAction(reset, options),

ambari-server/src/main/python/ambari_server/serverConfiguration.py

@@ -179,6 +179,8 @@ class ServerConfigDefaults(object):
     self.JDK_INSTALL_DIR = ""
     self.JDK_SEARCH_PATTERN = ""
     self.JAVA_EXE_SUBPATH = ""
+    self.JDK_SECURITY_DIR = "jre/lib/security"
+    self.SERVER_RESOURCES_DIR = "/var/lib/ambari-server/resources"
 
     # Configuration defaults
     self.DEFAULT_CONF_DIR = ""

ambari-server/src/main/python/ambari_server/serverSetup.py

@@ -72,8 +72,7 @@ JDK_PROMPT = "[{0}] {1}\n"
 JDK_CUSTOM_CHOICE_PROMPT = "[{0}] - Custom JDK\n==============================================================================\nEnter choice ({1}): "
 JDK_VALID_CHOICES = "^[{0}{1:d}]$"
 
-JDK_INDEX = 0
-
+IS_CUSTOM_JDK = False
 
 def get_supported_jdbc_drivers():
   factory = DBMSConfigFactory()
@@ -329,6 +328,7 @@ class JDKSetup(object):
   # Downloads and installs the JDK and the JCE policy archive
   #
   def download_and_install_jdk(self, args):
+    global IS_CUSTOM_JDK
     properties = get_ambari_properties()
     if properties == -1:
       err = "Error getting ambari properties"
@@ -393,6 +393,7 @@ class JDKSetup(object):
     )
 
     if jdk_num == str(custom_jdk_number):
+      IS_CUSTOM_JDK = True
       print_warning_msg("JDK must be installed on all hosts and JAVA_HOME must be valid on all hosts.")
       print_warning_msg(jcePolicyWarn)
       args.java_home = get_validated_string_input("Path to JAVA_HOME: ", None, None, None, False, False)
@@ -871,6 +872,43 @@ def extract_views():
   return 0
 
 
+def unpack_jce_policy():
+  properties = get_ambari_properties()
+  jdk_path = properties.get_property(JAVA_HOME_PROPERTY)
+  jdk_security_path = jdk_path + os.sep + configDefaults.JDK_SECURITY_DIR
+
+  jce_name = properties.get_property(JCE_NAME_PROPERTY)
+  jce_zip_path = configDefaults.SERVER_RESOURCES_DIR + os.sep + jce_name
+  f = None
+
+  import zipfile
+  if os.path.exists(jdk_security_path) and os.path.exists(jce_zip_path):
+    try:
+      f = zipfile.ZipFile(jce_zip_path, "r")
+      zip_members = f.namelist()
+      unziped_jce_path = os.path.split(zip_members[len(zip_members) - 1])[0]
+      f.extractall(jdk_security_path)
+    finally:
+      try:
+        f.close()
+      except Exception as e:
+        err = "Fail during the extraction of {0}.".format(jce_zip_path)
+        raise FatalException(1, err)
+  else:
+    err = "The path {0} or {1} is invalid.".format(jdk_security_path, jce_zip_path)
+    raise FatalException(1, err)
+
+  if unziped_jce_path:
+    from_path = jdk_security_path + os.sep + unziped_jce_path
+    jce_files = os.listdir(from_path)
+    for i in range(len(jce_files)):
+      jce_files[i] = from_path + os.sep + jce_files[i]
+    from ambari_commons.os_utils import copy_files
+    copy_files(jce_files, jdk_security_path)
+    dir_to_delete = jdk_security_path + os.sep + unziped_jce_path.split(os.sep)[0]
+    shutil.rmtree(dir_to_delete)
+  return 0
+
 #
 # Setup the Ambari Server.
 #
@@ -912,6 +950,14 @@ def setup(options):
     err = 'Downloading or installing JDK failed: {0}. Exiting.'.format(e)
     raise FatalException(e.code, err)
 
+  if not IS_CUSTOM_JDK: # If it's not a custom JDK, will also install JCE policy automatically
+    print 'Installing JCE policy...'
+    try:
+      unpack_jce_policy()
+    except FatalException as e:
+      err = 'Installing JCE failed: {0}. Exiting.'.format(e)
+      raise FatalException(e.code, err)
+
   print 'Completing setup...'
   retcode = configure_os_settings()
   if not retcode == 0:
@@ -937,6 +983,42 @@ def setup(options):
   adjust_directory_permissions(read_ambari_user())
 
 
+#
+# Setup the JCE policy for Ambari Server.
+#
+def setup_jce_policy(args):
+  if os.path.exists(args[1]):
+    if not os.path.split(args[1])[0] == configDefaults.SERVER_RESOURCES_DIR:
+      try:
+        shutil.copy(args[1], configDefaults.SERVER_RESOURCES_DIR)
+      except Exception as e:
+        err = "Fail while trying to copy {0} to {1}. {2}".format(args[1], configDefaults.SERVER_RESOURCES_DIR, e)
+        raise FatalException(1, err)
+  else:
+    err = "Can not run 'setup-jce'. Invalid path {0}.".format(args[1])
+    raise FatalException(1, err)
+
+  from ambari_commons.os_utils import search_file
+  from ambari_server.serverConfiguration import AMBARI_PROPERTIES_FILE, get_conf_dir
+  conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
+  properties = get_ambari_properties()
+  zip_name = os.path.split(args[1])[1]
+  properties.process_pair(JCE_NAME_PROPERTY, zip_name)
+  try:
+    properties.store(open(conf_file, "w"))
+  except Exception, e:
+    print_error_msg('Could not write ambari config file "%s": %s' % (conf_file, e))
+
+  print 'Installing JCE policy...'
+  try:
+    unpack_jce_policy()
+  except FatalException as e:
+    err = 'Installing JCE failed: {0}. Exiting.'.format(e)
+    raise FatalException(e.code, err)
+  print 'NOTE: Restart Ambari Server to apply changes' + \
+        ' ("ambari-server restart|stop|start")'
+
+
 #
 # Resets the Ambari Server.
 #

ambari-server/src/main/python/ambari_server/setupActions.py

@@ -36,3 +36,4 @@ ENCRYPT_PASSWORDS_ACTION = "encrypt-passwords"
 SETUP_SECURITY_ACTION = "setup-security"
 BACKUP_ACTION = "backup"
 RESTORE_ACTION = "restore"
+SETUP_JCE_ACTION = "setup-jce"

ambari-server/src/test/python/TestAmbariServer.py

@@ -76,7 +76,7 @@ with patch("platform.linux_distribution", return_value = os_distro_value):
         from ambari_server.serverUtils import is_server_runing, refresh_stack_hash
         from ambari_server.serverSetup import check_selinux, check_ambari_user, proceedJDBCProperties, SE_STATUS_DISABLED, SE_MODE_ENFORCING, configure_os_settings, \
           download_and_install_jdk, prompt_db_properties, setup, \
-          AmbariUserChecks, AmbariUserChecksLinux, AmbariUserChecksWindows, JDKSetup, reset
+          AmbariUserChecks, AmbariUserChecksLinux, AmbariUserChecksWindows, JDKSetup, reset, setup_jce_policy, unpack_jce_policy
         from ambari_server.serverUpgrade import upgrade, upgrade_local_repo, change_objects_owner, upgrade_stack, \
           run_stack_upgrade, run_metainfo_upgrade, run_schema_upgrade, move_user_custom_actions
         from ambari_server.setupHttps import is_valid_https_port, setup_https, import_cert_and_key_action, get_fqdn, \
@@ -2658,6 +2658,125 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
     self.assertEqual(result, "one")
     pass
 
+
+  @patch("ambari_server.serverSetup.get_ambari_properties")
+  @patch("os.path.exists")
+  @patch("zipfile.ZipFile")
+  @patch("os.path.split")
+  @patch("os.listdir")
+  @patch("ambari_commons.os_utils.copy_files")
+  @patch("shutil.rmtree")
+  def test_unpack_jce_policy(self, rmtree_mock, copy_files_mock, os_listdir_mock, os_path_split_mock, zipfile_mock, exists_mock, get_ambari_properties_mock):
+
+    # Testing the case when the zip file doesn't contains any folder
+    properties = MagicMock()
+    get_ambari_properties_mock.return_value = properties
+    exists_mock.return_value = True
+    zipfile = MagicMock()
+    zipfile_mock.return_value = zipfile
+    zip_members = ["US_export_policy.jar", "local_policy.jar", "README.txt"]
+    zipfile.namelist.return_value = zip_members
+    os_path_split_mock.return_value = [""]
+
+    unpack_jce_policy()
+    self.assertTrue(get_ambari_properties_mock.called)
+    self.assertTrue(exists_mock.called)
+    self.assertTrue(zipfile_mock.called)
+    self.assertTrue(os_path_split_mock.called)
+
+    # Testing the case when the zip file contains a folder
+    unziped_jce_path = "jce"
+    os_path_split_mock.return_value = unziped_jce_path
+
+    unpack_jce_policy()
+    self.assertTrue(get_ambari_properties_mock.called)
+    self.assertTrue(exists_mock.called)
+    self.assertTrue(zipfile_mock.called)
+    self.assertTrue(os_listdir_mock.called)
+    self.assertTrue(copy_files_mock.called)
+    self.assertTrue(rmtree_mock.called)
+
+    # Testing when the jdk_security_path or jce_zip_path doesn't exist
+    exists_mock.return_value = False
+    try:
+      unpack_jce_policy()
+    except FatalException:
+      self.assertTrue(True)
+    exists_mock.return_value = True
+
+    # Testing when zipfile fail with an error
+    zipfile_mock.side_effect = FatalException(1,"Extract error")
+    try:
+      unpack_jce_policy()
+    except FatalException:
+      self.assertTrue(True)
+
+
+  @patch("os.path.exists")
+  @patch("shutil.copy")
+  @patch("ambari_server.serverSetup.os.path.split")
+  @patch("ambari_server.serverSetup.unpack_jce_policy")
+  @patch("ambari_server.serverSetup.get_ambari_properties")
+  @patch("ambari_commons.os_utils.search_file")
+  @patch("__builtin__.open")
+  def test_setup_jce_policy(self, open_mock, search_file_mock, get_ambari_properties_mock, unpack_jce_policy_mock, path_split_mock, shutil_copy_mock, exists_mock):
+    exists_mock.return_value = True
+    properties = MagicMock()
+    unpack_jce_policy_mock.return_value = 0
+    get_ambari_properties_mock.return_value = properties
+    conf_file = 'etc/ambari-server/conf/ambari.properties'
+    search_file_mock.return_value = conf_file
+    path_split_mock.return_value = ["/path/to", "JCEPolicy.zip"]
+    args = ['setup-jce', '/path/to/JCEPolicy.zip']
+
+    setup_jce_policy(args)
+    shutil_copy_mock.assert_called_with(args[1], configDefaults.SERVER_RESOURCES_DIR)
+    self.assertTrue(unpack_jce_policy_mock.called)
+    self.assertTrue(get_ambari_properties_mock.called)
+    self.assertTrue(properties.store.called)
+
+    # Testing that if the source and the destination is the same will not try to copy the file
+    path_split_mock.return_value = [configDefaults.SERVER_RESOURCES_DIR, "JCEPolicy.zip"]
+    shutil_copy_mock.reset_mock()
+
+    setup_jce_policy(args)
+    self.assertFalse(shutil_copy_mock.called)
+    self.assertTrue(unpack_jce_policy_mock.called)
+    self.assertTrue(get_ambari_properties_mock.called)
+    self.assertTrue(properties.store.called)
+    path_split_mock.return_value = ["/path/to", "JCEPolicy.zip"]
+
+    # Testing with bad path
+    exists_mock.return_value = False
+    try:
+      setup_jce_policy(args)
+    except FatalException:
+      self.assertTrue(True)
+    exists_mock.return_value = True
+
+    # Testing with an error produced by shutil.copy
+    shutil_copy_mock.reset_mock()
+    shutil_copy_mock.side_effect = FatalException(1, "Error trying to copy the file.")
+    try:
+      setup_jce_policy(args)
+    except FatalException:
+      self.assertTrue(True)
+
+    # Testing with an error produced by Properties.store function
+    properties.store.side_effect = Exception("Invalid file.")
+    try:
+      setup_jce_policy(args)
+    except Exception:
+      self.assertTrue(True)
+    properties.reset_mock()
+
+    # Testing with an error produced by unpack_jce_policy
+    unpack_jce_policy_mock.side_effect = FatalException(1, "Can not install JCE policy")
+    try:
+      setup_jce_policy(args)
+    except FatalException:
+      self.assertTrue(True)
+
   @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
   @patch("ambari_commons.firewall.run_os_command")
   @patch("os.path.exists")
@@ -2688,7 +2807,8 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
   @patch("ambari_server.serverSetup.extract_views")
   @patch("ambari_server.serverSetup.adjust_directory_permissions")
   @patch("ambari_server.serverSetup.read_ambari_user")
-  def test_setup(self, read_ambari_user_mock, adjust_dirs_mock, extract_views_mock, proceedJDBCProperties_mock, is_root_mock,
+  @patch("ambari_server.serverSetup.unpack_jce_policy")
+  def test_setup(self, unpack_jce_policy_mock, read_ambari_user_mock, adjust_dirs_mock, extract_views_mock, proceedJDBCProperties_mock, is_root_mock,
                  disable_security_enhancements_mock, check_jdbc_drivers_mock, check_ambari_user_mock,
                  download_jdk_mock, configure_os_settings_mock, get_ambari_properties_mock,
                  get_YN_input_mock, gvsi_mock, gvsi_1_mock,
@@ -2737,6 +2857,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
     check_postgre_up_mock.return_value = (PGConfig.PG_STATUS_RUNNING, 0, "", "")
     configure_postgres_mock.return_value = (0, "", "")
     run_os_command_1_mock.return_value = (0, "", "")
+    unpack_jce_policy_mock.return_value = 0
 
     def reset_mocks():
       is_jdbc_user_changed_mock.reset_mock()
@@ -4514,7 +4635,8 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
   @patch("ambari_server.serverSetup.configure_os_settings")
   @patch('__builtin__.raw_input')
   @patch("ambari_server.serverSetup.disable_security_enhancements")
-  def test_setup_remote_db_wo_client(self, check_selinux_mock, raw_input, configure_os_settings_mock,
+  @patch("ambari_server.serverSetup.unpack_jce_policy")
+  def test_setup_remote_db_wo_client(self, unpack_jce_policy_mock, check_selinux_mock, raw_input, configure_os_settings_mock,
                                      download_jdk_mock, check_ambari_user_mock, is_root_mock, check_jdbc_drivers_mock,
                                      read_password_mock, ensure_jdbc_driver_installed_mock, store_remote_properties_mock,
                                      get_validated_string_input_0_mock, get_YN_input_0_mock,
@@ -4552,6 +4674,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
     download_jdk_mock.return_value = 0
     configure_os_settings_mock.return_value = 0
     verify_setup_allowed_method.return_value = 0
+    unpack_jce_policy_mock.return_value = 0
 
     try:
       setup(args)
@@ -6049,7 +6172,8 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
   @patch("ambari_server.serverSetup.adjust_directory_permissions")
   @patch("sys.exit")
   @patch("__builtin__.raw_input")
-  def test_ambariServerSetupWithCustomDbName(self, raw_input, exit_mock, adjust_dirs_mock, extract_views_mock, store_password_file_mock,
+  @patch("ambari_server.serverSetup.unpack_jce_policy")
+  def test_ambariServerSetupWithCustomDbName(self, unpack_jce_policy_mock, raw_input, exit_mock, adjust_dirs_mock, extract_views_mock, store_password_file_mock,
                                              get_is_secure_mock, setup_db_mock, is_root_mock, #is_local_database_mock,
                                              check_selinux_mock, check_jdbc_drivers_mock, check_ambari_user_mock,
                                              check_postgre_up_mock, configure_postgres_mock,