AMBARI-14376. Create KerberosHelper method to create the headless Kerberos identities and keytab files (inline) (rlevas)

ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java

@@ -234,6 +234,23 @@ public interface KerberosHelper {
                                                                   Set<String> services)
       throws KerberosInvalidConfigurationException, AmbariException;
 
+  /**
+   * Ensures that the relevant headless (or user) Kerberos identities are created and cached.
+   *
+   * This can be called any number of times and only the missing identities will be created.
+   * @param cluster                the cluster
+   * @param existingConfigurations the cluster's existing configurations
+   * @param services               the set of services to process
+   * @return
+   * @throws AmbariException
+   * @throws KerberosInvalidConfigurationException if an issue occurs trying to get the
+   *                                               Kerberos-specific configuration details
+   */
+  boolean ensureHeadlessIdentities(Cluster cluster,
+                                   Map<String, Map<String, String>> existingConfigurations,
+                                   Set<String> services)
+      throws KerberosInvalidConfigurationException, AmbariException;
+
   /**
    * Create a unique identity to use for testing the general Kerberos configuration.
    *

ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java

@@ -34,10 +34,10 @@ import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.google.inject.Injector;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.Role;
 import org.apache.ambari.server.RoleCommand;
-import org.apache.ambari.server.ServiceComponentNotFoundException;
 import org.apache.ambari.server.actionmanager.ActionManager;
 import org.apache.ambari.server.actionmanager.RequestFactory;
 import org.apache.ambari.server.actionmanager.Stage;
@@ -60,6 +60,7 @@ import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
 import org.apache.ambari.server.controller.utilities.ClusterControllerHelper;
 import org.apache.ambari.server.controller.utilities.PredicateBuilder;
 import org.apache.ambari.server.metadata.RoleCommandOrder;
+import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO;
 import org.apache.ambari.server.security.authorization.AuthorizationException;
 import org.apache.ambari.server.security.SecurePasswordHelper;
 import org.apache.ambari.server.security.credential.Credential;
@@ -110,12 +111,14 @@ import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory;
 import org.apache.ambari.server.state.kerberos.KerberosIdentityDescriptor;
 import org.apache.ambari.server.state.kerberos.KerberosKeytabDescriptor;
 import org.apache.ambari.server.state.kerberos.KerberosPrincipalDescriptor;
+import org.apache.ambari.server.state.kerberos.KerberosPrincipalType;
 import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor;
 import org.apache.ambari.server.state.kerberos.VariableReplacementHelper;
 import org.apache.ambari.server.state.svccomphost.ServiceComponentHostServerActionEvent;
 import org.apache.ambari.server.utils.StageUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.directory.server.kerberos.shared.keytab.Keytab;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -186,6 +189,16 @@ public class KerberosHelperImpl implements KerberosHelper {
   @Inject
   private SecurePasswordHelper securePasswordHelper;
 
+  @Inject
+  private KerberosPrincipalDAO kerberosPrincipalDAO;
+
+  /**
+   * The injector used to create new instances of helper classes like CreatePrincipalsServerAction
+   * and CreateKeytabFilesServerAction.
+   */
+  @Inject
+  private Injector injector;
+
   /**
    * The secure storage facility to use to store KDC administrator credential.
    */
@@ -357,6 +370,57 @@ public class KerberosHelperImpl implements KerberosHelper {
     return kerberosConfigurations;
   }
 
+  @Override
+  public boolean ensureHeadlessIdentities(Cluster cluster, Map<String, Map<String, String>> existingConfigurations, Set<String> services)
+      throws KerberosInvalidConfigurationException, AmbariException {
+
+    KerberosDetails kerberosDetails = getKerberosDetails(cluster, null);
+
+    // Only perform this task if Ambari manages Kerberos identities
+    if (kerberosDetails.manageIdentities()) {
+      KerberosDescriptor kerberosDescriptor = getKerberosDescriptor(cluster);
+
+      Map<String, String> kerberosDescriptorProperties = kerberosDescriptor.getProperties();
+      Map<String, Map<String, String>> configurations = addAdditionalConfigurations(cluster,
+          deepCopy(existingConfigurations), null, kerberosDescriptorProperties);
+
+      Map<String, String> kerberosConfiguration = kerberosDetails.getKerberosEnvProperties();
+      KerberosOperationHandler kerberosOperationHandler = kerberosOperationHandlerFactory.getKerberosOperationHandler(kerberosDetails.getKdcType());
+
+      for (String serviceName : services) {
+        // Set properties...
+        KerberosServiceDescriptor serviceDescriptor = kerberosDescriptor.getService(serviceName);
+
+        if (serviceDescriptor != null) {
+          Map<String, KerberosComponentDescriptor> componentDescriptors = serviceDescriptor.getComponents();
+          for (KerberosComponentDescriptor componentDescriptor : componentDescriptors.values()) {
+            if (componentDescriptor != null) {
+              List<KerberosIdentityDescriptor> identityDescriptors;
+
+              // Handle the service-level Kerberos identities
+              identityDescriptors = serviceDescriptor.getIdentities(true);
+              if (identityDescriptors != null) {
+                for (KerberosIdentityDescriptor identityDescriptor : identityDescriptors) {
+                  createUserIdentity(identityDescriptor, kerberosConfiguration, kerberosOperationHandler, configurations);
+                }
+              }
+
+              // Handle the component-level Kerberos identities
+              identityDescriptors = componentDescriptor.getIdentities(true);
+              if (identityDescriptors != null) {
+                for (KerberosIdentityDescriptor identityDescriptor : identityDescriptors) {
+                  createUserIdentity(identityDescriptor, kerberosConfiguration, kerberosOperationHandler, configurations);
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+
+    return true;
+  }
+
   @Override
   public RequestStageContainer createTestIdentity(Cluster cluster, Map<String, String> commandParamsStage,
                                                   RequestStageContainer requestStageContainer)
@@ -916,6 +980,77 @@ public class KerberosHelperImpl implements KerberosHelper {
     }
   }
 
+  /**
+   * Creates the principal and cached keytab file for the specified identity, if it is determined to
+   * be user (or headless) identity
+   * <p/>
+   * If the identity is determined not to be a user identity, it is skipped.
+   *
+   * @param identityDescriptor       the Kerberos identity to process
+   * @param kerberosEnvProperties    the kerberos-env properties
+   * @param kerberosOperationHandler the relevant KerberosOperationHandler
+   * @param configurations           the existing configurations for the cluster
+   * @return true if the identity was created; otherwise false
+   * @throws AmbariException
+   */
+  private boolean createUserIdentity(KerberosIdentityDescriptor identityDescriptor,
+                                     Map<String, String> kerberosEnvProperties,
+                                     KerberosOperationHandler kerberosOperationHandler,
+                                     Map<String, Map<String, String>> configurations)
+      throws AmbariException {
+
+    boolean created = false;
+
+    if (identityDescriptor != null) {
+      KerberosPrincipalDescriptor principalDescriptor = identityDescriptor.getPrincipalDescriptor();
+
+      if (principalDescriptor != null) {
+        // If this principal indicates it is a user principal, continue, else skip it.
+        if (KerberosPrincipalType.USER == principalDescriptor.getType()) {
+          String principal = variableReplacementHelper.replaceVariables(principalDescriptor.getValue(), configurations);
+
+          // If this principal is already in the Ambari database, then don't try to recreate it or it's
+          // keytab file.
+          if (!kerberosPrincipalDAO.exists(principal)) {
+            CreatePrincipalsServerAction.CreatePrincipalResult result;
+
+            result = injector.getInstance(CreatePrincipalsServerAction.class).createPrincipal(
+                principal,
+                false,
+                kerberosEnvProperties,
+                kerberosOperationHandler,
+                null);
+
+            if (result == null) {
+              throw new AmbariException("Failed to create the account for " + principal);
+            } else {
+              KerberosKeytabDescriptor keytabDescriptor = identityDescriptor.getKeytabDescriptor();
+
+              if (keytabDescriptor != null) {
+                Keytab keytab = injector.getInstance(CreateKeytabFilesServerAction.class).createKeytab(
+                    principal,
+                    result.getPassword(),
+                    result.getKeyNumber(),
+                    kerberosOperationHandler,
+                    true,
+                    true,
+                    null);
+
+                if (keytab == null) {
+                  throw new AmbariException("Failed to create the keytab for " + principal);
+                }
+              }
+
+              created = true;
+            }
+          }
+        }
+      }
+    }
+
+    return created;
+  }
+
   /**
    * Validate the KDC admin credentials.
    *

ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java

@@ -28,6 +28,7 @@ import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO;
 import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO;
 import org.apache.ambari.server.orm.entities.HostEntity;
 import org.apache.ambari.server.orm.entities.KerberosPrincipalEntity;
+import org.apache.ambari.server.serveraction.ActionLog;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.directory.server.kerberos.shared.keytab.Keytab;
 import org.slf4j.Logger;
@@ -161,8 +162,7 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
         message = "The data directory has not been set. Generated keytab files can not be stored.";
         LOG.error(message);
         commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
-      }
-      else {
+      } else {
         Map<String, String> principalPasswordMap = getPrincipalPasswordMap(requestSharedDataContext);
         Map<String, Integer> principalKeyNumberMap = getPrincipalKeyNumberMap(requestSharedDataContext);
 
@@ -177,6 +177,7 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
             // Look up the current evaluatedPrincipal's password.
             // If found create the keytab file, else try to find it in the cache.
             String password = principalPasswordMap.get(evaluatedPrincipal);
+            Integer keyNumber = principalKeyNumberMap.get(evaluatedPrincipal);
 
             message = String.format("Creating keytab file for %s on host %s", evaluatedPrincipal, hostName);
             LOG.info(message);
@@ -230,66 +231,9 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
                   }
                 }
               } else {
-                Keytab keytab = null;
-
-                // Possibly get the keytab from the cache
-                if (visitedPrincipalKeys != null) {
-                  // Since we have visited this principal before, attempt to pull the keytab from the
-                  // cache...
-                  KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal);
-                  String cachedKeytabPath = (principalEntity == null) ? null : principalEntity.getCachedKeytabPath();
-
-                  if (cachedKeytabPath != null) {
-                    try {
-                      keytab = Keytab.read(new File(cachedKeytabPath));
-                    } catch (IOException e) {
-                      message = String.format("Failed to read the cached keytab for %s, recreating if possible - %s",
-                          evaluatedPrincipal, e.getMessage());
-
-                      if (LOG.isDebugEnabled()) {
-                        LOG.warn(message, e);
-                      } else {
-                        LOG.warn(message, e);
-                      }
-                    }
-                  }
-                }
+                boolean canCache = ("true".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.KEYTAB_FILE_IS_CACHABLE)));
 
-                // If the keytab was not retrieved from the cache... create it.
-                if (keytab == null) {
-                  Integer keyNumber = principalKeyNumberMap.get(evaluatedPrincipal);
-
-                  try {
-                    keytab = operationHandler.createKeytab(evaluatedPrincipal, password, keyNumber);
-
-                    // If the current identity does not represent a service, copy it to a secure location
-                    // and store that location so it can be reused rather than recreate it.
-                    KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal);
-                    if (principalEntity != null) {
-                      if (!principalEntity.isService() && ("true".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.KEYTAB_FILE_IS_CACHABLE)))) {
-                        File cachedKeytabFile = cacheKeytab(evaluatedPrincipal, keytab);
-                        String previousCachedFilePath = principalEntity.getCachedKeytabPath();
-                        String cachedKeytabFilePath = ((cachedKeytabFile == null) || !cachedKeytabFile.exists())
-                            ? null
-                            : cachedKeytabFile.getAbsolutePath();
-
-                        principalEntity.setCachedKeytabPath(cachedKeytabFilePath);
-                        kerberosPrincipalDAO.merge(principalEntity);
-
-                        if(previousCachedFilePath != null) {
-                          if(!new File(previousCachedFilePath).delete()) {
-                            LOG.debug(String.format("Failed to remove orphaned cache file %s", previousCachedFilePath));
-                          }
-                        }
-                      }
-                    }
-                  } catch (KerberosOperationException e) {
-                    message = String.format("Failed to create keytab file for %s - %s", evaluatedPrincipal, e.getMessage());
-                    actionLog.writeStdErr(message);
-                    LOG.error(message, e);
-                    commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
-                  }
-                }
+                Keytab keytab = createKeytab(evaluatedPrincipal, password, keyNumber, operationHandler, visitedPrincipalKeys != null, canCache, actionLog);
 
                 if (keytab != null) {
                   try {
@@ -310,7 +254,16 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
                     LOG.error(message, e);
                     commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
                   }
+                } else {
+                  commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
+                }
+
+                if (visitedPrincipalKeys == null) {
+                  visitedPrincipalKeys = new HashSet<String>();
+                  visitedIdentities.put(evaluatedPrincipal, visitedPrincipalKeys);
                 }
+
+                visitedPrincipalKeys.add(visitationKey);
               }
             } else {
               message = String.format("Failed to create keytab file for %s, the container directory does not exist: %s",
@@ -319,24 +272,95 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
               LOG.error(message);
               commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
             }
+          } else {
+            LOG.debug(String.format("Skipping previously processed keytab for %s on host %s", evaluatedPrincipal, hostName));
+          }
+        }
+      }
+    }
 
-            if(visitedPrincipalKeys == null) {
-              visitedPrincipalKeys = new HashSet<String>();
-              visitedIdentities.put(evaluatedPrincipal, visitedPrincipalKeys);
-            }
+    return commandReport;
+  }
 
-            visitedPrincipalKeys.add(visitationKey);
+  /**
+   * Creates the keytab or gets one from the cache for a principal.
+   *
+   * @param principal        the principal name for the Keytab to create
+   * @param password         the password for the Keytab to create
+   * @param keyNumber        the key number for the Keytab to create
+   * @param operationHandler the KerberosOperationHandler for the relevant KDC
+   * @param checkCache       true to check the cache for an existing Keytab; otherwise false
+   * @param canCache         true to cache the resulting keytab (if generated); otherwise false
+   * @param actionLog        the logger (may be null if no logging is desired)
+   * @return a Keytab
+   * @throws AmbariException
+   */
+  public Keytab createKeytab(String principal, String password, Integer keyNumber,
+                             KerberosOperationHandler operationHandler, boolean checkCache,
+                             boolean canCache, ActionLog actionLog) throws AmbariException {
+    Keytab keytab = null;
+
+    // Possibly get the keytab from the cache
+    if (checkCache) {
+      // Attempt to pull the keytab from the cache...
+      KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(principal);
+      String cachedKeytabPath = (principalEntity == null) ? null : principalEntity.getCachedKeytabPath();
+
+      if (cachedKeytabPath != null) {
+        try {
+          keytab = Keytab.read(new File(cachedKeytabPath));
+        } catch (IOException e) {
+          String message = String.format("Failed to read the cached keytab for %s, recreating if possible - %s",
+              principal, e.getMessage());
+
+          if (LOG.isDebugEnabled()) {
+            LOG.warn(message, e);
+          } else {
+            LOG.warn(message, e);
           }
-          else {
-            LOG.debug(String.format("Skipping previously processed keytab for %s on host %s", evaluatedPrincipal, hostName));
+        }
+      }
+    }
+
+    // If the keytab was not retrieved from the cache... create it.
+    if (keytab == null) {
+      try {
+        keytab = operationHandler.createKeytab(principal, password, keyNumber);
+
+        // If the current identity does not represent a service, copy it to a secure location
+        // and store that location so it can be reused rather than recreate it.
+        KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(principal);
+        if (principalEntity != null) {
+          if (!principalEntity.isService() && canCache) {
+            File cachedKeytabFile = cacheKeytab(principal, keytab);
+            String previousCachedFilePath = principalEntity.getCachedKeytabPath();
+            String cachedKeytabFilePath = ((cachedKeytabFile == null) || !cachedKeytabFile.exists())
+                ? null
+                : cachedKeytabFile.getAbsolutePath();
+
+            principalEntity.setCachedKeytabPath(cachedKeytabFilePath);
+            kerberosPrincipalDAO.merge(principalEntity);
+
+            if (previousCachedFilePath != null) {
+              if (!new File(previousCachedFilePath).delete()) {
+                LOG.debug(String.format("Failed to remove orphaned cache file %s", previousCachedFilePath));
+              }
+            }
           }
         }
+      } catch (KerberosOperationException e) {
+        String message = String.format("Failed to create keytab file for %s - %s", principal, e.getMessage());
+        if (actionLog != null) {
+          actionLog.writeStdErr(message);
+        }
+        LOG.error(message, e);
       }
     }
 
-    return commandReport;
+    return keytab;
   }
 
+
   /**
    * Cache a keytab given its relative principal name and the keytab data.
    * <p/>

ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java

@@ -25,6 +25,7 @@ import org.apache.ambari.server.agent.CommandReport;
 import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO;
 import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO;
 import org.apache.ambari.server.security.SecurePasswordHelper;
+import org.apache.ambari.server.serveraction.ActionLog;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -119,91 +120,126 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
       String password = principalPasswordMap.get(evaluatedPrincipal);
 
       if (password == null) {
-        String message = String.format("Creating principal, %s", evaluatedPrincipal);
-        LOG.info(message);
-        actionLog.writeStdOut(message);
-
-        Integer length;
-        Integer minLowercaseLetters;
-        Integer minUppercaseLetters;
-        Integer minDigits;
-        Integer minPunctuation;
-        Integer minWhitespace;
-
-        if(kerberosConfiguration == null) {
-          length = null;
-          minLowercaseLetters= null;
-          minUppercaseLetters= null;
-          minDigits= null;
-          minPunctuation= null;
-          minWhitespace= null;
+        boolean servicePrincipal = "service".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.PRINCIPAL_TYPE));
+        CreatePrincipalResult result = createPrincipal(evaluatedPrincipal, servicePrincipal, kerberosConfiguration, operationHandler, actionLog);
+
+        if(result == null) {
+          commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
         }
         else {
-          length = toInt(kerberosConfiguration.get("password_length"));
-          minLowercaseLetters = toInt(kerberosConfiguration.get("password_min_lowercase_letters"));
-          minUppercaseLetters = toInt(kerberosConfiguration.get("password_min_uppercase_letters"));
-          minDigits = toInt(kerberosConfiguration.get("password_min_digits"));
-          minPunctuation = toInt(kerberosConfiguration.get("password_min_punctuation"));
-          minWhitespace = toInt(kerberosConfiguration.get("password_min_whitespace"));
+          principalPasswordMap.put(evaluatedPrincipal, result.getPassword());
+          principalKeyNumberMap.put(evaluatedPrincipal, result.getKeyNumber());
         }
+      }
+    }
 
-        password = securePasswordHelper.createSecurePassword(length, minLowercaseLetters, minUppercaseLetters, minDigits, minPunctuation, minWhitespace);
-
-        try {
-          boolean servicePrincipal = "service".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.PRINCIPAL_TYPE));
-
-          if (operationHandler.principalExists(evaluatedPrincipal)) {
-            // Create a new password since we need to know what it is.
-            // A new password/key would have been generated after exporting the keytab anyways.
-            message = String.format("Principal, %s, already exists, setting new password", evaluatedPrincipal);
-            LOG.warn(message);
-            actionLog.writeStdOut(message);
-            Integer keyNumber = operationHandler.setPrincipalPassword(evaluatedPrincipal, password);
-
-            if (keyNumber != null) {
-              principalPasswordMap.put(evaluatedPrincipal, password);
-              principalKeyNumberMap.put(evaluatedPrincipal, keyNumber);
-              message = String.format("Successfully set password for %s", evaluatedPrincipal);
-              LOG.debug(message);
-            } else {
-              message = String.format("Failed to set password for %s - unknown reason", evaluatedPrincipal);
-              LOG.error(message);
-              actionLog.writeStdErr(message);
-              commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
-            }
-          } else {
-            message = String.format("Creating new principal, %s", evaluatedPrincipal);
-            LOG.debug(message);
-
-            Integer keyNumber = operationHandler.createPrincipal(evaluatedPrincipal, password, servicePrincipal);
-
-            if (keyNumber != null) {
-              principalPasswordMap.put(evaluatedPrincipal, password);
-              principalKeyNumberMap.put(evaluatedPrincipal, keyNumber);
-              message = String.format("Successfully created new principal, %s", evaluatedPrincipal);
-              LOG.debug(message);
-            } else {
-              message = String.format("Failed to create principal, %s - unknown reason", evaluatedPrincipal);
-              LOG.error(message);
-              actionLog.writeStdErr(message);
-              commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
-            }
-          }
+    return commandReport;
+  }
+
+  /**
+   * Creates a principal in the relevant KDC
+   *
+   * @param principal                the principal name to create
+   * @param isServicePrincipal       true if the principal is a service principal; false if the
+   *                                 principal is a user principal
+   * @param kerberosConfiguration    the kerberos-env configuration properties
+   * @param kerberosOperationHandler the KerberosOperationHandler for the relevant KDC
+   * @param actionLog                the logger (may be null if no logging is desired)
+   * @return a CreatePrincipalResult containing the generated password and key number value
+   */
+  public CreatePrincipalResult createPrincipal(String principal, boolean isServicePrincipal,
+                                               Map<String, String> kerberosConfiguration,
+                                               KerberosOperationHandler kerberosOperationHandler,
+                                               ActionLog actionLog) {
+    CreatePrincipalResult result = null;
+
+    String message = String.format("Creating principal, %s", principal);
+    LOG.info(message);
+    if(actionLog != null) {
+      actionLog.writeStdOut(message);
+    }
+
+    Integer length;
+    Integer minLowercaseLetters;
+    Integer minUppercaseLetters;
+    Integer minDigits;
+    Integer minPunctuation;
+    Integer minWhitespace;
+
+    if(kerberosConfiguration == null) {
+      length = null;
+      minLowercaseLetters= null;
+      minUppercaseLetters= null;
+      minDigits= null;
+      minPunctuation= null;
+      minWhitespace= null;
+    }
+    else {
+      length = toInt(kerberosConfiguration.get("password_length"));
+      minLowercaseLetters = toInt(kerberosConfiguration.get("password_min_lowercase_letters"));
+      minUppercaseLetters = toInt(kerberosConfiguration.get("password_min_uppercase_letters"));
+      minDigits = toInt(kerberosConfiguration.get("password_min_digits"));
+      minPunctuation = toInt(kerberosConfiguration.get("password_min_punctuation"));
+      minWhitespace = toInt(kerberosConfiguration.get("password_min_whitespace"));
+    }
+
+    String password = securePasswordHelper.createSecurePassword(length, minLowercaseLetters, minUppercaseLetters, minDigits, minPunctuation, minWhitespace);
+
+    try {
+
+      if (kerberosOperationHandler.principalExists(principal)) {
+        // Create a new password since we need to know what it is.
+        // A new password/key would have been generated after exporting the keytab anyways.
+        message = String.format("Principal, %s, already exists, setting new password", principal);
+        LOG.warn(message);
+        if(actionLog != null) {
+          actionLog.writeStdOut(message);
+        }
+
+        Integer keyNumber = kerberosOperationHandler.setPrincipalPassword(principal, password);
 
-          if (!kerberosPrincipalDAO.exists(evaluatedPrincipal)) {
-            kerberosPrincipalDAO.create(evaluatedPrincipal, servicePrincipal);
+        if (keyNumber != null) {
+          message = String.format("Successfully set password for %s", principal);
+          LOG.debug(message);
+        } else {
+          message = String.format("Failed to set password for %s - unknown reason", principal);
+          LOG.error(message);
+          if(actionLog != null) {
+            actionLog.writeStdErr(message);
           }
+        }
+      } else {
+        message = String.format("Creating new principal, %s", principal);
+        LOG.debug(message);
 
-        } catch (KerberosOperationException e) {
-          message = String.format("Failed to create principal, %s - %s", evaluatedPrincipal, e.getMessage());
-          LOG.error(message, e);
-          actionLog.writeStdErr(message);
-          commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
+        Integer keyNumber = kerberosOperationHandler.createPrincipal(principal, password, isServicePrincipal);
+
+        if (keyNumber != null) {
+          result = new CreatePrincipalResult(principal, password, keyNumber);
+          message = String.format("Successfully created new principal, %s", principal);
+          LOG.debug(message);
+        } else {
+          message = String.format("Failed to create principal, %s - unknown reason", principal);
+          LOG.error(message);
+          if(actionLog != null) {
+            actionLog.writeStdErr(message);
+          }
         }
       }
+
+      if (!kerberosPrincipalDAO.exists(principal)) {
+        kerberosPrincipalDAO.create(principal, isServicePrincipal);
+      }
+
+    } catch (KerberosOperationException e) {
+      message = String.format("Failed to create principal, %s - %s", principal, e.getMessage());
+      LOG.error(message, e);
+      if(actionLog != null) {
+        actionLog.writeStdErr(message);
+      }
     }
 
-    return commandReport;
+    return result;
   }
 
   /**
@@ -215,7 +251,7 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
    * @param string the string to parse
    * @return an Integer or null
    */
-  private Integer toInt(String string) {
+  private static Integer toInt(String string) {
     if ((string == null) || string.isEmpty()) {
       return null;
     } else {
@@ -226,4 +262,53 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
       }
     }
   }
+
+  /**
+   * CreatePrincipalResult holds values created as a result of creating a principal in a KDC.
+   */
+  public static class CreatePrincipalResult {
+    final private String principal;
+    final private String password;
+    final private Integer keyNumber;
+
+    /**
+     * Constructor
+     *
+     * @param principal a principal name
+     * @param password  a password
+     * @param keyNumber a key number
+     */
+    public CreatePrincipalResult(String principal, String password, Integer keyNumber) {
+      this.principal = principal;
+      this.password = password;
+      this.keyNumber = keyNumber;
+    }
+
+    /**
+     * Gets the principal name
+     *
+     * @return the principal name
+     */
+    public String getPrincipal() {
+      return principal;
+    }
+
+    /**
+     * Gets the principal's password
+     *
+     * @return the principal's passwrod
+     */
+    public String getPassword() {
+      return password;
+    }
+
+    /**
+     * Gets the password's key number
+     *
+     * @return the password's key number
+     */
+    public Integer getKeyNumber() {
+      return keyNumber;
+    }
+  }
 }

ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java

@@ -61,6 +61,9 @@ import org.apache.ambari.server.security.credential.PrincipalKeyCredential;
 import org.apache.ambari.server.security.encryption.CredentialStoreService;
 import org.apache.ambari.server.security.encryption.CredentialStoreServiceImpl;
 import org.apache.ambari.server.security.encryption.CredentialStoreType;
+import org.apache.ambari.server.serveraction.ActionLog;
+import org.apache.ambari.server.serveraction.kerberos.CreateKeytabFilesServerAction;
+import org.apache.ambari.server.serveraction.kerberos.CreatePrincipalsServerAction;
 import org.apache.ambari.server.serveraction.kerberos.KDCType;
 import org.apache.ambari.server.serveraction.kerberos.KerberosConfigDataFileWriterFactory;
 import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurationException;
@@ -99,7 +102,9 @@ import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor;
 import org.apache.ambari.server.state.stack.OsFamily;
 import org.apache.ambari.server.topology.TopologyManager;
 import org.apache.ambari.server.utils.StageUtils;
+import org.apache.directory.server.kerberos.shared.keytab.Keytab;
 import org.easymock.Capture;
+import org.easymock.CaptureType;
 import org.easymock.EasyMockSupport;
 import org.easymock.IAnswer;
 import org.junit.After;
@@ -193,6 +198,8 @@ public class KerberosHelperTest extends EasyMockSupport {
         bind(StackManagerFactory.class).toInstance(createNiceMock(StackManagerFactory.class));
         bind(KerberosHelper.class).to(KerberosHelperImpl.class);
         bind(CredentialStoreService.class).to(CredentialStoreServiceImpl.class);
+        bind(CreatePrincipalsServerAction.class).toInstance(createMock(CreatePrincipalsServerAction.class));
+        bind(CreateKeytabFilesServerAction.class).toInstance(createMock(CreateKeytabFilesServerAction.class));
       }
     });
 
@@ -1978,17 +1985,17 @@ public class KerberosHelperTest extends EasyMockSupport {
     expect(krb5ConfConfig.getProperties()).andReturn(krb5ConfProperties).atLeastOnce();
 
     final KerberosPrincipalDescriptor principalDescriptor1 = createMockPrincipalDescriptor(
-        "service1/_HOST@${realm}", "service1user", "service1-site/service.kerberos.principal");
+        "service1/_HOST@${realm}", KerberosPrincipalType.SERVICE, "service1user", "service1-site/service.kerberos.principal");
     final KerberosPrincipalDescriptor principalDescriptor1a = createMockPrincipalDescriptor(
-        "component1a/_HOST@${realm}", "service1user", "service1-site/component1a.kerberos.principal");
+        "component1a/_HOST@${realm}", KerberosPrincipalType.SERVICE, "service1user", "service1-site/component1a.kerberos.principal");
     final KerberosPrincipalDescriptor principalDescriptor1b = createMockPrincipalDescriptor(
-        "component1b/_HOST@${realm}", "service1user", "service1-site/component1b.kerberos.principal");
+        "component1b/_HOST@${realm}", KerberosPrincipalType.SERVICE, "service1user", "service1-site/component1b.kerberos.principal");
     final KerberosPrincipalDescriptor principalDescriptor2a = createMockPrincipalDescriptor(
-        "component2a/_HOST@${realm}", "service2user", "service2-site/component2a.kerberos.principal");
+        "component2a/_HOST@${realm}", KerberosPrincipalType.SERVICE, "service2user", "service2-site/component2a.kerberos.principal");
     final KerberosPrincipalDescriptor principalDescriptor2b = createMockPrincipalDescriptor(
-        "component2b/_HOST@${realm}", "service2user", "service2-site/component2b.kerberos.principal");
+        "component2b/_HOST@${realm}", KerberosPrincipalType.SERVICE, "service2user", "service2-site/component2b.kerberos.principal");
     final KerberosPrincipalDescriptor principalDescriptor3a = createMockPrincipalDescriptor(
-        "component3a/_HOST@${realm}", "service3user", "service3-site/component3a.kerberos.principal");
+        "component3a/_HOST@${realm}", KerberosPrincipalType.SERVICE, "service3user", "service3-site/component3a.kerberos.principal");
 
     final KerberosKeytabDescriptor keytabDescriptor1 = createMockKeytabDescriptor(
         "keytab1", "service1-site/service.kerberos.keytab");
@@ -2249,7 +2256,7 @@ public class KerberosHelperTest extends EasyMockSupport {
     injector.getInstance(AmbariMetaInfo.class).init();
 
     Map<String, Map<String, String>> updates1 = kerberosHelper.getServiceConfigurationUpdates(
-        cluster, existingConfigurations,        new HashSet<String>(Arrays.asList("SERVICE1", "SERVICE2", "SERVICE3")));
+        cluster, existingConfigurations, new HashSet<String>(Arrays.asList("SERVICE1", "SERVICE2", "SERVICE3")));
 
     Map<String, Map<String, String>> updates2 = kerberosHelper.getServiceConfigurationUpdates(
         cluster, existingConfigurations, new HashSet<String>(Arrays.asList("SERVICE1", "SERVICE3")));
@@ -2346,6 +2353,162 @@ public class KerberosHelperTest extends EasyMockSupport {
     assertEquals(expectedExistingConfigurations, existingConfigurations);
   }
 
+  @Test
+  public void testEnsureHeadlessIdentities() throws Exception {
+    Map<String, String> propertiesKrb5Conf = new HashMap<String, String>();
+
+    Map<String, String> propertiesKerberosEnv = new HashMap<String, String>();
+    propertiesKerberosEnv.put("realm", "EXAMPLE.COM");
+    propertiesKerberosEnv.put("kdc_type", "mit-kdc");
+    propertiesKerberosEnv.put("password_length", "20");
+    propertiesKerberosEnv.put("password_min_lowercase_letters", "1");
+    propertiesKerberosEnv.put("password_min_uppercase_letters", "1");
+    propertiesKerberosEnv.put("password_min_digits", "1");
+    propertiesKerberosEnv.put("password_min_punctuation", "0");
+    propertiesKerberosEnv.put("password_min_whitespace","0");
+
+    Config configKrb5Conf = createMock(Config.class);
+    expect(configKrb5Conf.getProperties()).andReturn(propertiesKrb5Conf).times(1);
+
+    Config configKerberosEnv = createMock(Config.class);
+    expect(configKerberosEnv.getProperties()).andReturn(propertiesKerberosEnv).times(1);
+
+    Host host1 = createMockHost("host1");
+    Host host2 = createMockHost("host3");
+    Host host3 = createMockHost("host2");
+
+    Map<String, ServiceComponentHost> service1Component1HostMap = new HashMap<String, ServiceComponentHost>();
+    service1Component1HostMap.put("host1", createMockServiceComponentHost());
+
+    Map<String, ServiceComponentHost> service2Component1HostMap = new HashMap<String, ServiceComponentHost>();
+    service2Component1HostMap.put("host2", createMockServiceComponentHost());
+
+    Map<String, ServiceComponent> service1ComponentMap = new HashMap<String, ServiceComponent>();
+    service1ComponentMap.put("COMPONENT11", createMockComponent("COMPONENT11", true, service1Component1HostMap));
+
+    Map<String, ServiceComponent> service2ComponentMap = new HashMap<String, ServiceComponent>();
+    service2ComponentMap.put("COMPONENT21", createMockComponent("COMPONENT21", true, service2Component1HostMap));
+
+    Service service1 = createMockService("SERVICE1", service1ComponentMap);
+    Service service2 = createMockService("SERVICE2", service2ComponentMap);
+
+    Map<String, Service> servicesMap = new HashMap<String, Service>();
+    servicesMap.put("SERVICE1", service1);
+    servicesMap.put("SERVICE2", service2);
+
+    Cluster cluster = createMock(Cluster.class);
+    expect(cluster.getDesiredConfigByType("krb5-conf")).andReturn(configKrb5Conf).times(1);
+    expect(cluster.getDesiredConfigByType("kerberos-env")).andReturn(configKerberosEnv).times(1);
+    expect(cluster.getSecurityType()).andReturn(SecurityType.KERBEROS).times(1);
+    expect(cluster.getCurrentStackVersion()).andReturn(new StackId("HDP", "2.2")).times(1);
+    expect(cluster.getClusterName()).andReturn("c1").times(2);
+    expect(cluster.getHosts()).andReturn(Arrays.asList(host1, host2, host3)).times(1);
+    expect(cluster.getServices()).andReturn(servicesMap).times(1);
+
+    Map<String, String> kerberosDescriptorProperties = new HashMap<String, String>();
+    kerberosDescriptorProperties.put("additional_realms", "");
+    kerberosDescriptorProperties.put("keytab_dir", "/etc/security/keytabs");
+    kerberosDescriptorProperties.put("realm", "${kerberos-env/realm}");
+
+    ArrayList<KerberosIdentityDescriptor> service1Component1Identities = new ArrayList<KerberosIdentityDescriptor>();
+    service1Component1Identities.add(createMockIdentityDescriptor(
+        "s1c1_1.user",
+        createMockPrincipalDescriptor("s1c1_1@${realm}", KerberosPrincipalType.USER, "s1c1", null),
+        createMockKeytabDescriptor("s1c1_1.user.keytab", null)
+    ));
+    service1Component1Identities.add(createMockIdentityDescriptor(
+        "s1c1_1.service",
+        createMockPrincipalDescriptor("s1c1_1/_HOST@${realm}", KerberosPrincipalType.SERVICE, "s1c1", null),
+        createMockKeytabDescriptor("s1c1_1.service.keytab", null)
+    ));
+
+    HashMap<String, KerberosComponentDescriptor> service1ComponentDescriptorMap = new HashMap<String, KerberosComponentDescriptor>();
+    service1ComponentDescriptorMap.put("COMPONENT11", createMockComponentDescriptor("COMPONENT11", service1Component1Identities, null));
+
+    List<KerberosIdentityDescriptor> service1Identities = new ArrayList<KerberosIdentityDescriptor>();
+    service1Identities.add(createMockIdentityDescriptor(
+        "s1_1.user",
+        createMockPrincipalDescriptor("s1_1@${realm}", KerberosPrincipalType.USER, "s1", null),
+        createMockKeytabDescriptor("s1_1.user.keytab", null)
+    ));
+    service1Identities.add(createMockIdentityDescriptor(
+        "s1_1.service",
+        createMockPrincipalDescriptor("s1/_HOST@${realm}", KerberosPrincipalType.SERVICE, "s1", null),
+        createMockKeytabDescriptor("s1.service.keytab", null)
+    ));
+
+    KerberosServiceDescriptor service1KerberosDescriptor = createMockServiceDescriptor("SERVICE1", service1ComponentDescriptorMap, service1Identities);
+
+    ArrayList<KerberosIdentityDescriptor> service2Component1Identities = new ArrayList<KerberosIdentityDescriptor>();
+    service2Component1Identities.add(createMockIdentityDescriptor(
+        "s2_1.user",
+        createMockPrincipalDescriptor("s2_1@${realm}", KerberosPrincipalType.USER, "s2", null),
+        createMockKeytabDescriptor("s2_1.user.keytab", null)
+    ));
+    service2Component1Identities.add(createMockIdentityDescriptor(
+        "s2c1_1.service",
+        createMockPrincipalDescriptor("s2c1_1/_HOST@${realm}", KerberosPrincipalType.SERVICE, "s2c1", null),
+        createMockKeytabDescriptor("s2c1_1.service.keytab", null)
+    ));
+
+    HashMap<String, KerberosComponentDescriptor> service2ComponentDescriptorMap = new HashMap<String, KerberosComponentDescriptor>();
+    service2ComponentDescriptorMap.put("COMPONENT21", createMockComponentDescriptor("COMPONENT21", service2Component1Identities, null));
+
+    KerberosServiceDescriptor service2KerberosDescriptor = createMockServiceDescriptor("SERVICE2", service2ComponentDescriptorMap, null);
+
+    KerberosDescriptor kerberosDescriptor = createMock(KerberosDescriptor.class);
+    expect(kerberosDescriptor.getProperties()).andReturn(kerberosDescriptorProperties);
+    expect(kerberosDescriptor.getService("SERVICE1")).andReturn(service1KerberosDescriptor).times(1);
+    expect(kerberosDescriptor.getService("SERVICE2")).andReturn(service2KerberosDescriptor).times(1);
+
+    setupGetDescriptorFromStack(kerberosDescriptor);
+
+    Map<String, Map<String, String>> existingConfigurations = new HashMap<String, Map<String, String>>();
+    existingConfigurations.put("kerberos-env", propertiesKerberosEnv);
+
+    Set<String> services = new HashSet<String>() {
+      {
+        add("SERVICE1");
+        add("SERVICE2");
+      }
+    };
+
+    Capture<? extends String> capturePrincipal = newCapture(CaptureType.ALL);
+    Capture<? extends String> capturePrincipalForKeytab = newCapture(CaptureType.ALL);
+
+    CreatePrincipalsServerAction createPrincipalsServerAction = injector.getInstance(CreatePrincipalsServerAction.class);
+    expect(createPrincipalsServerAction.createPrincipal(capture(capturePrincipal), eq(false), anyObject(Map.class),  anyObject(KerberosOperationHandler.class), isNull(ActionLog.class)))
+        .andReturn(new CreatePrincipalsServerAction.CreatePrincipalResult("anything", "password", 1))
+        .times(3);
+
+    CreateKeytabFilesServerAction createKeytabFilesServerAction = injector.getInstance(CreateKeytabFilesServerAction.class);
+    expect(createKeytabFilesServerAction.createKeytab(capture(capturePrincipalForKeytab), eq("password"), eq(1), anyObject(KerberosOperationHandler.class), eq(true), eq(true), isNull(ActionLog.class)))
+        .andReturn(new Keytab())
+        .times(3);
+    
+    replayAll();
+
+    AmbariMetaInfo ambariMetaInfo = injector.getInstance(AmbariMetaInfo.class);
+    ambariMetaInfo.init();
+
+    KerberosHelper kerberosHelper = injector.getInstance(KerberosHelper.class);
+    kerberosHelper.ensureHeadlessIdentities(cluster, existingConfigurations, services);
+
+    verifyAll();
+
+    List<? extends String> capturedPrincipals = capturePrincipal.getValues();
+    assertEquals(3, capturedPrincipals.size());
+    assertTrue(capturedPrincipals.contains("s1_1@EXAMPLE.COM"));
+    assertTrue(capturedPrincipals.contains("s1c1_1@EXAMPLE.COM"));
+    assertTrue(capturedPrincipals.contains("s2_1@EXAMPLE.COM"));
+
+    List<? extends String> capturedPrincipalsForKeytab = capturePrincipalForKeytab.getValues();
+    assertEquals(3, capturedPrincipalsForKeytab.size());
+    assertTrue(capturedPrincipalsForKeytab.contains("s1_1@EXAMPLE.COM"));
+    assertTrue(capturedPrincipalsForKeytab.contains("s1c1_1@EXAMPLE.COM"));
+    assertTrue(capturedPrincipalsForKeytab.contains("s2_1@EXAMPLE.COM"));
+  }
+
   private void setClusterController() throws Exception {
     KerberosHelper kerberosHelper = injector.getInstance(KerberosHelper.class);
 
@@ -2363,8 +2526,8 @@ public class KerberosHelperTest extends EasyMockSupport {
     Resource resource = createStrictMock(Resource.class);
     Set<Resource> result = Collections.singleton(resource);
 
-    Capture<Predicate> predicateCapture = new Capture<Predicate>();
-    Capture<Request> requestCapture = new Capture<Request>();
+    Capture<Predicate> predicateCapture = newCapture();
+    Capture<Request> requestCapture = newCapture();
 
     //todo: validate captures
 
@@ -2394,8 +2557,8 @@ public class KerberosHelperTest extends EasyMockSupport {
     ResourceProvider resourceProvider = createStrictMock(ResourceProvider.class);
     expect(clusterController.ensureResourceProvider(Resource.Type.Artifact)).andReturn(resourceProvider).once();
 
-    Capture<Predicate> predicateCapture = new Capture<Predicate>();
-    Capture<Request> requestCapture = new Capture<Request>();
+    Capture<Predicate> predicateCapture = newCapture();
+    Capture<Request> requestCapture = newCapture();
 
     //todo: validate captures
 
@@ -3575,11 +3738,11 @@ public class KerberosHelperTest extends EasyMockSupport {
   }
 
   private KerberosPrincipalDescriptor createMockPrincipalDescriptor(String value,
-                                                                    String localUsername,
+                                                                    KerberosPrincipalType type, String localUsername,
                                                                     String configuration) {
     KerberosPrincipalDescriptor descriptor = createMock(KerberosPrincipalDescriptor.class);
     expect(descriptor.getValue()).andReturn(value).anyTimes();
-    expect(descriptor.getType()).andReturn(KerberosPrincipalType.SERVICE).anyTimes();
+    expect(descriptor.getType()).andReturn(type).anyTimes();
     expect(descriptor.getLocalUsername()).andReturn(localUsername).anyTimes();
     expect(descriptor.getConfiguration()).andReturn(configuration).anyTimes();
     return descriptor;