AMBARI-19038. Support migration of LDAP users & groups to PAM (rlevas)

ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/groups/GroupsEditCtrl.js

@@ -68,6 +68,7 @@ angular.module('ambariAdminConsole')
 
   function loadMembers(){
     $scope.group.getMembers().then(function(members) {
+      $scope.group.groupTypeName = $t(GroupConstants.TYPES[$scope.group.group_type].LABEL_KEY);
       $scope.groupMembers = members;
       $scope.group.editingUsers = angular.copy($scope.groupMembers);
     });
@@ -81,6 +82,8 @@ angular.module('ambariAdminConsole')
     loadMembers();
   });
 
+  $scope.group.getGroupType();
+
   $scope.deleteGroup = function(group) {
     ConfirmationModal.show(
       $t('common.delete', {

ambari-server/pom.xml

@@ -1684,7 +1684,7 @@
     <dependency>
       <groupId>net.java.dev.jna</groupId>
       <artifactId>jna</artifactId>
-      <version>4.1.0</version>
+      <version>4.3.0</version>
     </dependency>
   </dependencies>
 

ambari-server/sbin/ambari-server

@@ -137,6 +137,10 @@ case "${1:-}" in
         echo -e "Setting up PAM properties..."
         $PYTHON "$AMBARI_PYTHON_EXECUTABLE" $@
         ;;
+  migrate-ldap-pam)
+        echo -e "Migration LDAP to PAM"
+        $PYTHON "$AMBARI_PYTHON_EXECUTABLE" $@
+        ;;
   setup-ldap)
         echo -e "Setting up LDAP properties..."
         $PYTHON "$AMBARI_PYTHON_EXECUTABLE" $@
@@ -203,7 +207,7 @@ case "${1:-}" in
         ;;
   *)
         echo "Usage: $AMBARI_EXECUTABLE
-        {start|stop|reset|restart|upgrade|status|upgradestack|setup|setup-jce|setup-ldap|sync-ldap|set-current|setup-security|refresh-stack-hash|backup|restore|update-host-names|check-database|enable-stack|setup-sso|db-purge-history|install-mpack|uninstall-mpack|upgrade-mpack|setup-kerberos} [options]
+        {start|stop|reset|restart|upgrade|status|upgradestack|setup|setup-jce|setup-ldap|sync-ldap|set-current|setup-security|refresh-stack-hash|backup|restore|update-host-names|check-database|enable-stack|setup-sso|db-purge-history|install-mpack|uninstall-mpack|upgrade-mpack|setup-kerberos|setup-pam|migrate-ldap-pam} [options]
         Use $AMBARI_PYTHON_EXECUTABLE <action> --help to get details on options available.
         Or, simply invoke ambari-server.py --help to print the options."
         exit 1

ambari-server/src/main/java/org/apache/ambari/server/security/authentication/LdapToPamMigrationHelper.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.security.authentication;
+
+import java.sql.SQLException;
+
+import org.apache.ambari.server.audit.AuditLoggerModule;
+import org.apache.ambari.server.controller.ControllerModule;
+import org.apache.ambari.server.orm.DBAccessor;
+import org.apache.ambari.server.orm.DBAccessor.DbType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.inject.Guice;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+
+public class LdapToPamMigrationHelper {
+  private static final Logger LOG = LoggerFactory.getLogger(LdapToPamMigrationHelper.class);
+
+  @Inject
+  private DBAccessor dbAccessor;
+
+  /**
+   * Migrate LDAP user & groups to PAM
+   *
+   * @throws SQLException if an error occurs while executing the needed SQL statements
+   */
+  private void migrateLdapUsersGroups() throws SQLException {
+    if (dbAccessor.getDbType() != DbType.ORACLE) { // Tested MYSQL, POSTGRES && MYSQL)
+      dbAccessor.executeQuery("UPDATE users SET user_type='PAM',ldap_user=0 WHERE ldap_user=1 and user_name not in (select user_name from (select user_name from users where user_type = 'PAM') as a)");
+      dbAccessor.executeQuery("UPDATE groups SET group_type='PAM',ldap_group=0 WHERE ldap_group=1 and group_name not in (select group_name from (select group_name from groups where group_type = 'PAM') as a)");
+    } else { // Tested ORACLE
+      dbAccessor.executeQuery("UPDATE users SET user_type='PAM',ldap_user=0 WHERE ldap_user=1 and user_name not in (select user_name from users where user_type = 'PAM')");
+      dbAccessor.executeQuery("UPDATE groups SET group_type='PAM',ldap_group=0 WHERE ldap_group=1 and group_name not in (select group_name from groups where group_type = 'PAM')");
+    }
+  }
+
+  /**
+   * Support changes needed to migrate LDAP users & groups to PAM
+   *
+   * @param args Simple key value json map
+   */
+  public static void main(String[] args) {
+
+    try {
+      Injector injector = Guice.createInjector(new ControllerModule(), new AuditLoggerModule());
+      LdapToPamMigrationHelper migrationHelper = injector.getInstance(LdapToPamMigrationHelper.class);
+
+      migrationHelper.migrateLdapUsersGroups();
+
+    } catch (Throwable t) {
+      LOG.error("Caught exception on migration. Exiting...", t);
+      System.exit(1);
+    }
+
+  }
+}

ambari-server/src/main/java/org/apache/ambari/server/security/authorization/Users.java

@@ -162,6 +162,10 @@ public class Users {
     if (userEntity != null) {
       userEntities.add(userEntity);
     }
+    userEntity = userDAO.findUserByNameAndType(userName, UserType.PAM);
+    if (userEntity != null) {
+      userEntities.add(userEntity);
+    }
     return (userEntities.isEmpty() || userEntities.size() > 1) ? null : new User(userEntities.get(0));
   }
 

ambari-server/src/main/python/ambari-server.py

@@ -52,8 +52,8 @@ from ambari_server.setupActions import BACKUP_ACTION, LDAP_SETUP_ACTION, LDAP_SY
   SETUP_ACTION, SETUP_SECURITY_ACTION,START_ACTION, STATUS_ACTION, STOP_ACTION, RESTART_ACTION, UPGRADE_ACTION, \
   SETUP_JCE_ACTION, SET_CURRENT_ACTION, START_ACTION, STATUS_ACTION, STOP_ACTION, UPGRADE_ACTION, \
   SETUP_JCE_ACTION, SET_CURRENT_ACTION, ENABLE_STACK_ACTION, SETUP_SSO_ACTION, \
-  DB_PURGE_ACTION, INSTALL_MPACK_ACTION, UNINSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION, PAM_SETUP_ACTION, KERBEROS_SETUP_ACTION
-from ambari_server.setupSecurity import setup_ldap, sync_ldap, setup_master_key, setup_ambari_krb5_jaas, setup_pam
+  DB_PURGE_ACTION, INSTALL_MPACK_ACTION, UNINSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION, PAM_SETUP_ACTION, MIGRATE_LDAP_PAM_ACTION, KERBEROS_SETUP_ACTION
+from ambari_server.setupSecurity import setup_ldap, sync_ldap, setup_master_key, setup_ambari_krb5_jaas, setup_pam, migrate_ldap_pam
 from ambari_server.userInput import get_validated_string_input
 from ambari_server.kerberos_setup import setup_kerberos
 
@@ -539,6 +539,11 @@ def init_ldap_setup_parser_options(parser):
   parser.add_option('--ldap-bind-anonym', default=None, help="Bind anonymously [true/false] for LDAP", dest="ldap_bind_anonym")
   parser.add_option('--ldap-sync-username-collisions-behavior', default=None, help="Handling behavior for username collisions [convert/skip] for LDAP sync", dest="ldap_sync_username_collisions_behavior")
 
+@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT)
+def init_pam_setup_parser_options(parser):
+  parser.add_option('--pam-config-file', default=None, help="Path to the PAM configuration file", dest="pam_config_file")
+  parser.add_option('--pam-auto-create-groups', default=None, help="Automatically create groups for authenticated users [true/false]", dest="pam_auto_create_groups")
+
 @OsFamilyFuncImpl(OsFamilyImpl.DEFAULT)
 def init_set_current_parser_options(parser):
   parser.add_option('--cluster-name', default=None, help="Cluster name", dest="cluster_name")
@@ -783,7 +788,8 @@ def create_user_action_map(args, options):
         INSTALL_MPACK_ACTION: UserAction(install_mpack, options),
         UNINSTALL_MPACK_ACTION: UserAction(uninstall_mpack, options),
         UPGRADE_MPACK_ACTION: UserAction(upgrade_mpack, options),
-        PAM_SETUP_ACTION: UserAction(setup_pam),
+        PAM_SETUP_ACTION: UserAction(setup_pam, options),
+        MIGRATE_LDAP_PAM_ACTION: UserAction(migrate_ldap_pam, options),
         KERBEROS_SETUP_ACTION: UserAction(setup_kerberos, options)
       }
   return action_map
@@ -814,7 +820,7 @@ def init_action_parser(action, parser):
     INSTALL_MPACK_ACTION: init_install_mpack_parser_options,
     UNINSTALL_MPACK_ACTION: init_uninstall_mpack_parser_options,
     UPGRADE_MPACK_ACTION: init_upgrade_mpack_parser_options,
-    PAM_SETUP_ACTION: init_empty_parser_options,
+    PAM_SETUP_ACTION: init_pam_setup_parser_options,
     KERBEROS_SETUP_ACTION: init_kerberos_setup_parser_options,
   }
   parser.add_option("-v", "--verbose",

ambari-server/src/main/python/ambari_server/setupActions.py

@@ -47,4 +47,5 @@ INSTALL_MPACK_ACTION = "install-mpack"
 UNINSTALL_MPACK_ACTION = "uninstall-mpack"
 UPGRADE_MPACK_ACTION = "upgrade-mpack"
 PAM_SETUP_ACTION = "setup-pam"
+MIGRATE_LDAP_PAM_ACTION = "migrate-ldap-pam"
 KERBEROS_SETUP_ACTION = "setup-kerberos"

ambari-server/src/main/python/ambari_server/setupSecurity.py

@@ -37,9 +37,9 @@ from ambari_commons.os_check import OSConst
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 from ambari_commons.os_utils import is_root, set_file_permissions, \
   run_os_command, search_file, is_valid_filepath, change_owner, get_ambari_repo_file_full_name, get_file_owner
-from ambari_server.serverConfiguration import configDefaults, \
+from ambari_server.serverConfiguration import configDefaults, parse_properties_file, \
   encrypt_password, find_jdk, find_properties_file, get_alias_string, get_ambari_properties, get_conf_dir, \
-  get_credential_store_location, get_is_persisted, get_is_secure, get_master_key_location, write_property, \
+  get_credential_store_location, get_is_persisted, get_is_secure, get_master_key_location, get_db_type, write_property, \
   get_original_master_key, get_value_from_properties, get_java_exe_path, is_alias_string, read_ambari_user, \
   read_passwd_for_alias, remove_password_file, save_passwd_for_alias, store_password_file, update_properties_2, \
   BLIND_PASSWORD, BOOTSTRAP_DIR_PROPERTY, IS_LDAP_CONFIGURED, JDBC_PASSWORD_FILENAME, JDBC_PASSWORD_PROPERTY, \
@@ -54,6 +54,8 @@ from ambari_server.serverUtils import is_server_runing, get_ambari_server_api_ba
 from ambari_server.setupActions import SETUP_ACTION, LDAP_SETUP_ACTION
 from ambari_server.userInput import get_validated_string_input, get_prompt_default, read_password, get_YN_input, quit_if_has_answer
 from ambari_server.serverClassPath import ServerClassPath
+from ambari_server.dbConfiguration import DBMSConfigFactory, check_jdbc_drivers, \
+  get_jdbc_driver_path, ensure_jdbc_driver_is_installed, LINUX_DBMS_KEYS_LIST
 
 logger = logging.getLogger(__name__)
 
@@ -64,6 +66,9 @@ REGEX_TRUE_FALSE = "^(true|false)?$"
 REGEX_SKIP_CONVERT = "^(skip|convert)?$"
 REGEX_REFERRAL = "^(follow|ignore)?$"
 REGEX_ANYTHING = ".*"
+LDAP_TO_PAM_MIGRATION_HELPER_CMD = "{0} -cp {1} " + \
+                                   "org.apache.ambari.server.security.authentication.LdapToPamMigrationHelper" + \
+                                   " >> " + configDefaults.SERVER_OUT_FILE + " 2>&1"
 
 CLIENT_SECURITY_KEY = "client.security"
 
@@ -621,8 +626,12 @@ def setup_ldap(options):
   properties = get_ambari_properties()
 
   if get_value_from_properties(properties,CLIENT_SECURITY_KEY,"") == 'pam':
-    err = "PAM is configured. Can not setup LDAP."
-    raise FatalException(1, err)
+    query = "PAM is currently configured, do you wish to use LDAP instead [y/n] (n)? "
+    if get_YN_input(query, False):
+      pass
+    else:
+      err = "PAM is configured. Can not setup LDAP."
+      raise FatalException(1, err)
 
   isSecure = get_is_secure(properties)
 
@@ -824,38 +833,112 @@ def ensure_can_start_under_current_user(ambari_user):
   return current_user
 
 class PamPropTemplate:
-  def __init__(self, properties, i_prop_name, i_prop_val_pattern, i_prompt_regex, i_allow_empty_prompt, i_prop_name_default=None):
+  def __init__(self, properties, i_option, i_prop_name, i_prop_val_pattern, i_prompt_regex, i_allow_empty_prompt, i_prop_name_default=None):
     self.prop_name = i_prop_name
+    self.option = i_option
     self.pam_prop_name = get_value_from_properties(properties, i_prop_name, i_prop_name_default)
     self.pam_prop_val_prompt = i_prop_val_pattern.format(get_prompt_default(self.pam_prop_name))
     self.prompt_regex = i_prompt_regex
     self.allow_empty_prompt = i_allow_empty_prompt
 
-def setup_pam():
+def init_pam_properties_list_reqd(properties, options):
+  properties = [
+    PamPropTemplate(properties, options.pam_config_file, PAM_CONFIG_FILE, "PAM configuration file* {0}: ", REGEX_ANYTHING, False, "/etc/pam.d/ambari"),
+    PamPropTemplate(properties, options.pam_auto_create_groups, AUTO_GROUP_CREATION, "Do you want to allow automatic group creation* [true/false] {0}: ", REGEX_TRUE_FALSE, False, "false"),
+  ]
+  return properties
+
+def setup_pam(options):
   if not is_root():
-    err = 'Ambari-server setup-pam should be run with ' \
-          'root-level privileges'
+    err = 'Ambari-server setup-pam should be run with root-level privileges'
     raise FatalException(4, err)
 
   properties = get_ambari_properties()
 
   if get_value_from_properties(properties,CLIENT_SECURITY_KEY,"") == 'ldap':
-    err = "LDAP is configured. Can not setup PAM."
-    raise FatalException(1, err)
+    query = "LDAP is currently configured, do you wish to use PAM instead [y/n] (n)? "
+    if get_YN_input(query, False):
+      pass
+    else:
+      err = "LDAP is configured. Can not setup PAM."
+      raise FatalException(1, err)
+
+  pam_property_list_reqd = init_pam_properties_list_reqd(properties, options)
 
   pam_property_value_map = {}
   pam_property_value_map[CLIENT_SECURITY_KEY] = 'pam'
 
-  pamConfig = get_validated_string_input("Enter PAM configuration file: ", PAM_CONFIG_FILE, REGEX_ANYTHING,
-                                         "Invalid characters in the input!", False, False)
-
-  pam_property_value_map[PAM_CONFIG_FILE] = pamConfig
+  for pam_prop in pam_property_list_reqd:
+    input = get_validated_string_input(pam_prop.pam_prop_val_prompt, pam_prop.pam_prop_name, pam_prop.prompt_regex,
+                                       "Invalid characters in the input!", False, pam_prop.allow_empty_prompt,
+                                       answer = pam_prop.option)
+    if input is not None and input != "":
+      pam_property_value_map[pam_prop.prop_name] = input
 
-  if get_YN_input("Do you want to allow automatic group creation [y/n] (y)? ", True):
-    pam_property_value_map[AUTO_GROUP_CREATION] = 'true'
-  else:
-    pam_property_value_map[AUTO_GROUP_CREATION] = 'false'
+  # Verify that the PAM config file exists, else show warning...
+  pam_config_file = pam_property_value_map[PAM_CONFIG_FILE]
+  if not os.path.exists(pam_config_file):
+    print_warning_msg("The PAM configuration file, {0} does not exist.  " \
+                      "Please create it before restarting Ambari.".format(pam_config_file))
 
   update_properties_2(properties, pam_property_value_map)
   print 'Saving...done'
   return 0
+
+#
+# Migration of LDAP users & groups to PAM
+#
+def migrate_ldap_pam(args):
+  properties = get_ambari_properties()
+
+  if get_value_from_properties(properties,CLIENT_SECURITY_KEY,"") != 'pam':
+    err = "PAM is not configured. Please configure PAM authentication first."
+    raise FatalException(1, err)
+
+  db_title = get_db_type(properties).title
+  confirm = get_YN_input("Ambari Server configured for %s. Confirm "
+                        "you have made a backup of the Ambari Server database [y/n] (y)? " % db_title, True)
+
+  if not confirm:
+    print_error_msg("Database backup is not confirmed")
+    return 1
+
+  jdk_path = get_java_exe_path()
+  if jdk_path is None:
+    print_error_msg("No JDK found, please run the \"setup\" "
+                    "command to install a JDK automatically or install any "
+                    "JDK manually to " + configDefaults.JDK_INSTALL_DIR)
+    return 1
+
+  # At this point, the args does not have the ambari database information.
+  # Augment the args with the correct ambari database information
+  parse_properties_file(args)
+
+  ensure_jdbc_driver_is_installed(args, properties)
+
+  print 'Migrating LDAP Users & Groups to PAM'
+
+  serverClassPath = ServerClassPath(properties, args)
+  class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell()
+
+  command = LDAP_TO_PAM_MIGRATION_HELPER_CMD.format(jdk_path, class_path)
+
+  ambari_user = read_ambari_user()
+  current_user = ensure_can_start_under_current_user(ambari_user)
+  environ = generate_env(args, ambari_user, current_user)
+
+  (retcode, stdout, stderr) = run_os_command(command, env=environ)
+  print_info_msg("Return code from LDAP to PAM migration command, retcode = " + str(retcode))
+  if stdout:
+    print "Console output from LDAP to PAM migration command:"
+    print stdout
+    print
+  if stderr:
+    print "Error output from LDAP to PAM migration command:"
+    print stderr
+    print
+  if retcode > 0:
+    print_error_msg("Error executing LDAP to PAM migration, please check the server logs.")
+  else:
+    print_info_msg('LDAP to PAM migration completed')
+  return retcode