AMBARI-15961. Kerberos wizard stuck trying to schedule service check operation (rlevas)

ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java

@@ -286,7 +286,7 @@ public class KerberosHelperImpl implements KerberosHelper {
       throws AmbariException, KerberosOperationException {
     return handle(cluster, getKerberosDetails(cluster, manageIdentities), serviceComponentFilter, hostFilter, identityFilter,
         hostsToForceKerberosOperations, requestStageContainer, new CreatePrincipalsAndKeytabsHandler(false, false,
-        false));
+            false));
   }
 
   @Override
@@ -433,8 +433,8 @@ public class KerberosHelperImpl implements KerberosHelper {
 
     return (applyStackAdvisorUpdates)
         ? applyStackAdvisorUpdates(cluster, services, configurations, kerberosConfigurations, propertiesToIgnore,
-      null, null,
-      kerberosEnabled)
+        null, null,
+        kerberosEnabled)
         : kerberosConfigurations;
   }
 
@@ -510,12 +510,12 @@ public class KerberosHelperImpl implements KerberosHelper {
       }
 
       StackAdvisorRequest request = StackAdvisorRequest.StackAdvisorRequestBuilder
-        .forStack(stackVersion.getStackName(), stackVersion.getStackVersion())
-        .forServices(new ArrayList<String>(services))
-        .forHosts(hostNames)
-        .withConfigurations(requestConfigurations)
-        .ofType(StackAdvisorRequest.StackAdvisorRequestType.CONFIGURATIONS)
-        .build();
+          .forStack(stackVersion.getStackName(), stackVersion.getStackVersion())
+          .forServices(new ArrayList<String>(services))
+          .forHosts(hostNames)
+          .withConfigurations(requestConfigurations)
+          .ofType(StackAdvisorRequest.StackAdvisorRequestType.CONFIGURATIONS)
+          .build();
 
       try {
         RecommendationResponse response = stackAdvisorHelper.recommend(request);
@@ -534,13 +534,13 @@ public class KerberosHelperImpl implements KerberosHelper {
             Set<String> ignoreProperties = (propertiesToIgnore == null) ? null : propertiesToIgnore.get(configType);
 
             addRecommendedPropertiesForConfigType(kerberosEnabled, kerberosConfigurations, configType,
-              recommendedConfigProperties,
-              existingConfigProperties, kerberosConfigProperties, ignoreProperties, propertiesToInsert);
+                recommendedConfigProperties,
+                existingConfigProperties, kerberosConfigProperties, ignoreProperties, propertiesToInsert);
             if (recommendedConfigPropertyAttributes != null) {
               removeRecommendedPropertiesForConfigType(kerberosEnabled, configType,
-                recommendedConfigPropertyAttributes,
-                existingConfigProperties,
-                kerberosConfigurations, ignoreProperties, propertiesToRemove);
+                  recommendedConfigPropertyAttributes,
+                  existingConfigProperties,
+                  kerberosConfigurations, ignoreProperties, propertiesToRemove);
             }
           }
         }
@@ -561,8 +561,8 @@ public class KerberosHelperImpl implements KerberosHelper {
                                                      String configType, Map<String, String> recommendedConfigProperties,
                                                      Map<String, String> existingConfigProperties,
                                                      Map<String, String> kerberosConfigProperties,
-                                                     Set<String> ignoreProperties, Map<String, Map<String, String>>
-                                                       propertiesToInsert) {
+                                                     Set<String> ignoreProperties,
+                                                     Map<String, Map<String, String>> propertiesToInsert) {
 
     for (Map.Entry<String, String> property : recommendedConfigProperties.entrySet()) {
       String propertyName = property.getKey();
@@ -575,8 +575,8 @@ public class KerberosHelperImpl implements KerberosHelper {
             // add the config and property if it also does not exist in the existing configurations
             if ((existingConfigProperties == null) || !existingConfigProperties.containsKey(propertyName)) {
               LOG.debug("Adding Kerberos configuration based on StackAdvisor recommendation:" +
-                  "\n\tConfigType: {}\n\tProperty: {}\n\tValue: {}",
-                configType, propertyName, recommendedValue);
+                      "\n\tConfigType: {}\n\tProperty: {}\n\tValue: {}",
+                  configType, propertyName, recommendedValue);
 
               HashMap<String, String> properties = new HashMap<String, String>();
               properties.put(propertyName, recommendedValue);
@@ -589,22 +589,22 @@ public class KerberosHelperImpl implements KerberosHelper {
               // add the property if it also does not exist in the existing configurations
               if ((existingConfigProperties == null) || !existingConfigProperties.containsKey(propertyName)) {
                 LOG.debug("Adding Kerberos configuration based on StackAdvisor recommendation:" +
-                    "\n\tConfigType: {}\n\tProperty: {}\n\tValue: {}",
-                  configType, propertyName, recommendedValue);
+                        "\n\tConfigType: {}\n\tProperty: {}\n\tValue: {}",
+                    configType, propertyName, recommendedValue);
 
                 kerberosConfigProperties.put(propertyName, recommendedValue);
               }
             } else if (!value.equals(recommendedValue)) {
               // If the recommended value is a change, automatically change it.
               LOG.debug("Updating Kerberos configuration based on StackAdvisor recommendation:" +
-                  "\n\tConfigType: {}\n\tProperty: {}\n\tOld Value: {}\n\tNew Value: {}",
-                configType, propertyName, value, recommendedValue);
+                      "\n\tConfigType: {}\n\tProperty: {}\n\tOld Value: {}\n\tNew Value: {}",
+                  configType, propertyName, value, recommendedValue);
 
               kerberosConfigProperties.put(propertyName, recommendedValue);
             }
           }
         } else if (propertiesToInsert != null && ((existingConfigProperties == null) || !existingConfigProperties
-          .containsKey(propertyName))) {
+            .containsKey(propertyName))) {
           Map<String, String> properties = propertiesToInsert.get(configType);
           if (properties == null) {
             properties = new HashMap<>();
@@ -612,8 +612,8 @@ public class KerberosHelperImpl implements KerberosHelper {
           }
 
           LOG.debug("Property to add to configuration based on StackAdvisor recommendation:" +
-              "\n\tConfigType: {}\n\tProperty: {}\n\tValue: {}",
-            configType, propertyName, recommendedValue);
+                  "\n\tConfigType: {}\n\tProperty: {}\n\tValue: {}",
+              configType, propertyName, recommendedValue);
 
           properties.put(propertyName, recommendedValue);
         }
@@ -633,7 +633,7 @@ public class KerberosHelperImpl implements KerberosHelper {
                                                         Map<String, String> existingConfigProperties,
                                                         Map<String, Map<String, String>> kerberosConfigurations,
                                                         Set<String> ignoreProperties, Map<String, Set<String>>
-                                                          propertiesToRemove) {
+                                                            propertiesToRemove) {
 
     for (Map.Entry<String, ValueAttributesInfo> property : recommendedConfigPropertyAttributes.entrySet()) {
       String propertyName = property.getKey();
@@ -642,12 +642,12 @@ public class KerberosHelperImpl implements KerberosHelper {
         // add to propertiesToBeRemoved map
         Map<String, String> kerberosConfigProperties = kerberosConfigurations.get(configType);
         if (((ignoreProperties == null) || !ignoreProperties.contains(propertyName)) &&
-          ((kerberosConfigProperties == null) || kerberosConfigProperties.get(propertyName) == null) &&
-          (existingConfigProperties != null && existingConfigProperties.containsKey(propertyName))) {
+            ((kerberosConfigProperties == null) || kerberosConfigProperties.get(propertyName) == null) &&
+            (existingConfigProperties != null && existingConfigProperties.containsKey(propertyName))) {
 
           LOG.debug("Property to remove from configuration based on StackAdvisor recommendation:" +
-              "\n\tConfigType: {}\n\tProperty: {}",
-            configType, propertyName);
+                  "\n\tConfigType: {}\n\tProperty: {}",
+              configType, propertyName);
 
           // kerberosEnabled add property to propertiesToRemove, otherwise to kerberosConfigurations map
           if (kerberosEnabled && propertiesToRemove != null) {
@@ -681,7 +681,7 @@ public class KerberosHelperImpl implements KerberosHelper {
 
       Map<String, String> kerberosDescriptorProperties = kerberosDescriptor.getProperties();
       Map<String, Map<String, String>> configurations = addAdditionalConfigurations(cluster,
-        deepCopy(existingConfigurations), null, kerberosDescriptorProperties);
+          deepCopy(existingConfigurations), null, kerberosDescriptorProperties);
 
       Map<String, String> kerberosConfiguration = kerberosDetails.getKerberosEnvProperties();
       KerberosOperationHandler kerberosOperationHandler = kerberosOperationHandlerFactory.getKerberosOperationHandler(kerberosDetails.getKdcType());
@@ -691,7 +691,7 @@ public class KerberosHelperImpl implements KerberosHelper {
         kerberosOperationHandler.open(administratorCredential, kerberosDetails.getDefaultRealm(), kerberosConfiguration);
       } catch (KerberosOperationException e) {
         String message = String.format("Failed to process the identities, could not properly open the KDC operation handler: %s",
-          e.getMessage());
+            e.getMessage());
         LOG.error(message);
         throw new AmbariException(message, e);
       }
@@ -734,7 +734,7 @@ public class KerberosHelperImpl implements KerberosHelper {
           configureAmbariIdentity(ambariServerIdentity, kerberosOperationHandler, configurations);
           try {
             KerberosChecker.checkJaasConfiguration();
-          } catch(AmbariException e) {
+          } catch (AmbariException e) {
             LOG.error("Error in Ambari JAAS configuration: ", e);
           }
         }
@@ -754,8 +754,8 @@ public class KerberosHelperImpl implements KerberosHelper {
   }
 
   private boolean configureAmbariIdentity(KerberosIdentityDescriptor ambariServerIdentity,
-                                          KerberosOperationHandler kerberosOperationHandler, Map<String, Map<String, String>>
-                                            configurations) throws AmbariException {
+                                          KerberosOperationHandler kerberosOperationHandler,
+                                          Map<String, Map<String, String>> configurations) throws AmbariException {
     boolean created = false;
 
 
@@ -772,8 +772,8 @@ public class KerberosHelperImpl implements KerberosHelper {
 
         if (keytabDescriptor != null) {
           injector.getInstance(ConfigureAmbariIndetityServerAction.class)
-            .createAndConfigureAmbariKeytab(principal, kerberosOperationHandler, keytabFilePath,
-              keytabDescriptor.getOwnerName(), keytabDescriptor.getOwnerAccess(), null);
+              .createAndConfigureAmbariKeytab(principal, kerberosOperationHandler, keytabFilePath,
+                  keytabDescriptor.getOwnerName(), keytabDescriptor.getOwnerAccess(), null);
           // throw new AmbariException("Failed to create the keytab for " + principal);
         }
 
@@ -788,7 +788,7 @@ public class KerberosHelperImpl implements KerberosHelper {
                                                   RequestStageContainer requestStageContainer)
       throws KerberosOperationException, AmbariException {
     return handleTestIdentity(cluster, getKerberosDetails(cluster, null), commandParamsStage, requestStageContainer,
-      new CreatePrincipalsAndKeytabsHandler(false, false, false));
+        new CreatePrincipalsAndKeytabsHandler(false, false, false));
   }
 
   @Override
@@ -815,7 +815,7 @@ public class KerberosHelperImpl implements KerberosHelper {
 
     boolean processAuthToLocalRules = true;
     Map<String, String> kerberosEnvProperties = existingConfigurations.get("kerberos-env");
-    if(kerberosEnvProperties.containsKey("manage_auth_to_local")) {
+    if (kerberosEnvProperties.containsKey("manage_auth_to_local")) {
       processAuthToLocalRules = Boolean.valueOf(kerberosEnvProperties.get("manage_auth_to_local"));
     }
 
@@ -856,37 +856,37 @@ public class KerberosHelperImpl implements KerberosHelper {
 
             Map<String, KerberosComponentDescriptor> components = service.getComponents();
             if (components != null) {
-              Map<String,ServiceComponent> serviceComponents = svc.getServiceComponents();
+              Map<String, ServiceComponent> serviceComponents = svc.getServiceComponents();
 
               for (KerberosComponentDescriptor component : components.values()) {
                 // When the cluster is provisioned by a Blueprint service components with
                 // cardinality 0+ might be left out from the Blueprint thus we have to check
                 // if they exist
                 ServiceComponent svcComp = null;
-                if (!serviceComponents.containsKey(component.getName()))
+                if (!serviceComponents.containsKey(component.getName())) {
                   continue;
+                }
 
                 svcComp = serviceComponents.get(component.getName());
 
                 boolean addSvcCompIdentities = false;
 
-                if (cluster.isBluePrintDeployed())
-                {
-                  if (svcComp.getDesiredState() == State.INSTALLED ||  svcComp.getDesiredState() == State.STARTED)
+                if (cluster.isBluePrintDeployed()) {
+                  if (svcComp.getDesiredState() == State.INSTALLED || svcComp.getDesiredState() == State.STARTED) {
                     addSvcCompIdentities = true;
-                }
-                else {
+                  }
+                } else {
 
                   // Since when the cluster is deployed through the UI ALL service components of the selected services are created
                   // with desired state INSTALLED regardless whether the service components were associated with hosts or not thus
                   // we can not determine if the component is installed or not.
                   // We rather look at service compoent hosts
-                  for (ServiceComponentHost svcCompHost: svcComp.getServiceComponentHosts().values()) {
+                  for (ServiceComponentHost svcCompHost : svcComp.getServiceComponentHosts().values()) {
                     if (svcCompHost.getDesiredState() != State.UNKNOWN
-                      && svcCompHost.getDesiredState() != State.UNINSTALLING
-                      && svcCompHost.getDesiredState() != State.UNINSTALLED
-                      && svcCompHost.getDesiredState() != State.INSTALL_FAILED
-                      && svcCompHost.getDesiredState() != State.WIPING_OUT) {
+                        && svcCompHost.getDesiredState() != State.UNINSTALLING
+                        && svcCompHost.getDesiredState() != State.UNINSTALLED
+                        && svcCompHost.getDesiredState() != State.INSTALL_FAILED
+                        && svcCompHost.getDesiredState() != State.WIPING_OUT) {
 
                       // If there is at least a host that contains the component add the identities
                       addSvcCompIdentities = true;
@@ -1300,7 +1300,7 @@ public class KerberosHelperImpl implements KerberosHelper {
                   KerberosPrincipalType principalType = principalDescriptor.getType();
 
                   // Assume the principal is a service principal if not specified
-                  if(principalType == null) {
+                  if (principalType == null) {
                     principalType = KerberosPrincipalType.SERVICE;
                   }
 
@@ -1402,6 +1402,7 @@ public class KerberosHelperImpl implements KerberosHelper {
                 false,
                 kerberosEnvProperties,
                 kerberosOperationHandler,
+                false,
                 null);
 
             if (result == null) {
@@ -1455,16 +1456,7 @@ public class KerberosHelperImpl implements KerberosHelper {
     if (kerberosDetails.manageIdentities()) {
       PrincipalKeyCredential credentials = getKDCAdministratorCredentials(cluster.getClusterName());
       if (credentials == null) {
-        throw new KerberosMissingAdminCredentialsException(
-            "Missing KDC administrator credentials.\n" +
-                "The KDC administrator credentials must be set as a persisted or temporary credential resource." +
-                "This may be done by issuing a POST to the /api/v1/clusters/:clusterName/credentials/kdc.admin.credential API entry point with the following payload:\n" +
-                "{\n" +
-                "  \"Credential\" : {\n" +
-                "    \"principal\" : \"(PRINCIPAL)\", \"key\" : \"(PASSWORD)\", \"type\" : \"(persisted|temporary)\"}\n" +
-                "  }\n" +
-                "}"
-        );
+        throw new KerberosMissingAdminCredentialsException();
       } else {
         KerberosOperationHandler operationHandler = kerberosOperationHandlerFactory.getKerberosOperationHandler(kerberosDetails.getKdcType());
 
@@ -1513,16 +1505,7 @@ public class KerberosHelperImpl implements KerberosHelper {
 
           // need to throw this outside of the try/catch so it isn't caught
           if (missingCredentials) {
-            throw new KerberosMissingAdminCredentialsException(
-                "Invalid KDC administrator credentials.\n" +
-                    "The KDC administrator credentials must be set as a persisted or temporary credential resource." +
-                    "This may be done by issuing a POST to the /api/v1/clusters/:clusterName/credentials/kdc.admin.credential API entry point with the following payload:\n" +
-                    "{\n" +
-                    "  \"Credential\" : {\n" +
-                    "    \"principal\" : \"(PRINCIPAL)\", \"key\" : \"(PASSWORD)\", \"type\" : \"(persisted|temporary)\"}\n" +
-                    "  }\n" +
-                    "}"
-            );
+            throw new KerberosMissingAdminCredentialsException();
           }
         }
       }
@@ -1715,192 +1698,145 @@ public class KerberosHelperImpl implements KerberosHelper {
         throw new AmbariException("The properties map must not be null.  It is needed to store data related to the service check identity");
       }
 
-      Map<String, Service> services = cluster.getServices();
+      List<ServiceComponentHost> serviceComponentHostsToProcess = new ArrayList<ServiceComponentHost>();
+      KerberosDescriptor kerberosDescriptor = getKerberosDescriptor(cluster);
+      KerberosIdentityDataFileWriter kerberosIdentityDataFileWriter = null;
 
-      if ((services != null) && !services.isEmpty()) {
-        String clusterName = cluster.getClusterName();
-        Map<String, Host> hosts = clusters.getHostsForCluster(clusterName);
-
-        if ((hosts != null) && !hosts.isEmpty()) {
-          List<ServiceComponentHost> serviceComponentHostsToProcess = new ArrayList<ServiceComponentHost>();
-          KerberosDescriptor kerberosDescriptor = getKerberosDescriptor(cluster);
-          KerberosIdentityDataFileWriter kerberosIdentityDataFileWriter = null;
-          Map<String, String> kerberosDescriptorProperties = kerberosDescriptor.getProperties();
-
-          // This is needed to help determine which hosts to perform actions for and create tasks for.
-          Set<String> hostsWithValidKerberosClient = getHostsWithValidKerberosClient(cluster);
-
-          // Create a temporary directory to store metadata needed to complete this task.  Information
-          // such as which principals and keytabs files to create as well as what configurations need
-          // to be update are stored in data files in this directory. Any keytab files are stored in
-          // this directory until they are distributed to their appropriate hosts.
-          File dataDirectory = createTemporaryDirectory();
-
-          // Create the file used to store details about principals and keytabs to create
-          File identityDataFile = new File(dataDirectory, KerberosIdentityDataFileWriter.DATA_FILE_NAME);
-
-          // Create a special identity for the test user
-          KerberosIdentityDescriptor identity = new KerberosIdentityDescriptor(new HashMap<String, Object>() {
-            {
-              put("principal",
-                  new HashMap<String, Object>() {
-                    {
-                      put("value", "${kerberos-env/service_check_principal_name}@${realm}");
-                      put("type", "user");
-                    }
-                  });
-              put("keytab",
-                  new HashMap<String, Object>() {
-                    {
-                      put("file", "${keytab_dir}/kerberos.service_check.${short_date}.keytab");
-
-                      put("owner", new HashMap<String, Object>() {{
-                        put("name", "${cluster-env/smokeuser}");
-                        put("access", "rw");
-                      }});
-
-                      put("group", new HashMap<String, Object>() {{
-                        put("name", "${cluster-env/user_group}");
-                        put("access", "r");
-                      }});
-
-                      put("cachable", "false");
-                    }
-                  });
-            }
-          });
+      Map<String, String> kerberosDescriptorProperties = kerberosDescriptor.getProperties();
 
-          try {
-            // Iterate over the hosts in the cluster to find the components installed in each.  For each
-            // component (aka service component host - sch) determine the configuration updates and
-            // and the principals an keytabs to create.
-            for (Host host : hosts.values()) {
-              String hostname = host.getHostName();
-
-              // Get a list of components on the current host
-              List<ServiceComponentHost> serviceComponentHosts = cluster.getServiceComponentHosts(hostname);
-
-              if ((serviceComponentHosts != null) && !serviceComponentHosts.isEmpty()) {
-                // Calculate the current host-specific configurations. These will be used to replace
-                // variables within the Kerberos descriptor data
-                Map<String, Map<String, String>> configurations = calculateConfigurations(cluster, hostname, kerberosDescriptorProperties);
-
-                // Add a short date value
-                configurations.get("").put("short_date", new SimpleDateFormat("MMddyy").format(new Date()));
-
-                // Iterate over the components installed on the current host to get the service and
-                // component-level Kerberos descriptors in order to determine which principals,
-                // keytab files, and configurations need to be created or updated.
-                for (ServiceComponentHost sch : serviceComponentHosts) {
-                  String serviceName = sch.getServiceName();
-                  String componentName = sch.getServiceComponentName();
-
-                  // If the current ServiceComponentHost represents the KERBEROS/KERBEROS_CLIENT and
-                  // indicates that the KERBEROS_CLIENT component is in the INSTALLED state, add the
-                  // current host to the set of hosts that should be handled...
-                  if (Service.Type.KERBEROS.name().equals(serviceName) &&
-                      Role.KERBEROS_CLIENT.name().equals(componentName) &&
-                      (sch.getState() == State.INSTALLED)) {
-                    hostsWithValidKerberosClient.add(hostname);
-                    int identitiesAdded = 0;
-
-                    // Lazily create the KerberosIdentityDataFileWriter instance...
-                    if (kerberosIdentityDataFileWriter == null) {
-                      kerberosIdentityDataFileWriter = kerberosIdentityDataFileWriterFactory.createKerberosIdentityDataFileWriter(identityDataFile);
-                    }
+      // This is needed to help determine which hosts to perform actions for and create tasks for.
+      Set<String> hostsWithValidKerberosClient = getHostsWithValidKerberosClient(cluster);
 
-                    // Add service-level principals (and keytabs)
-                    identitiesAdded += addIdentities(kerberosIdentityDataFileWriter, Collections.singleton(identity),
-                        null, hostname, serviceName, componentName, null, configurations);
+      // Create a temporary directory to store metadata needed to complete this task.  Information
+      // such as which principals and keytabs files to create as well as what configurations need
+      // to be update are stored in data files in this directory. Any keytab files are stored in
+      // this directory until they are distributed to their appropriate hosts.
+      File dataDirectory = createTemporaryDirectory();
 
-                    if (identitiesAdded > 0) {
-                      // Add the relevant principal name and keytab file data to the command params state
-                      if (!commandParameters.containsKey("principal_name") || !commandParameters.containsKey("keytab_file")) {
-                        commandParameters.put("principal_name",
-                            variableReplacementHelper.replaceVariables(identity.getPrincipalDescriptor().getValue(), configurations));
-                        commandParameters.put("keytab_file",
-                            variableReplacementHelper.replaceVariables(identity.getKeytabDescriptor().getFile(), configurations));
-                      }
+      // Create the file used to store details about principals and keytabs to create
+      File identityDataFile = new File(dataDirectory, KerberosIdentityDataFileWriter.DATA_FILE_NAME);
 
-                      serviceComponentHostsToProcess.add(sch);
-                    }
-                  }
-                }
-              }
-            }
-          } catch (IOException e) {
-            String message = String.format("Failed to write index file - %s", identityDataFile.getAbsolutePath());
-            LOG.error(message);
-            throw new AmbariException(message, e);
-          } catch (Exception e) {
-            // make sure to log what is going wrong
-            LOG.error("Failed " + e);
-            throw e;
-          } finally {
-            if (kerberosIdentityDataFileWriter != null) {
-              // Make sure the data file is closed
-              try {
-                kerberosIdentityDataFileWriter.close();
-              } catch (IOException e) {
-                LOG.warn("Failed to close the index file writer", e);
-              }
-            }
-          }
+      // Calculate the current non-host-specific configurations. These will be used to replace
+      // variables within the Kerberos descriptor data
+      Map<String, Map<String, String>> configurations = calculateConfigurations(cluster, null, kerberosDescriptorProperties);
 
-          // If there are ServiceComponentHosts to process, make sure the administrator credential
-          // are available
-          if (!serviceComponentHostsToProcess.isEmpty()) {
-            try {
-              validateKDCCredentials(kerberosDetails, cluster);
-            } catch (Exception e) {
-              LOG.error("Cannot validate credentials: " + e);
-              try {
-                FileUtils.deleteDirectory(dataDirectory);
-              } catch (Throwable t) {
-                LOG.warn(String.format("The data directory (%s) was not deleted due to an error condition - {%s}",
-                    dataDirectory.getAbsolutePath(), t.getMessage()), t);
-              }
+      String principal = variableReplacementHelper.replaceVariables("${kerberos-env/service_check_principal_name}@${realm}", configurations);
+      String principalType = "user";
+
+      String keytabFilePath = variableReplacementHelper.replaceVariables("${keytab_dir}/kerberos.service_check.${short_date}.keytab", configurations);
+      String keytabFileOwnerName = variableReplacementHelper.replaceVariables("${cluster-env/smokeuser}", configurations);
+      String keytabFileOwnerAccess = "rw";
+      String keytabFileGroupName = variableReplacementHelper.replaceVariables("${cluster-env/user_group}", configurations);
+      String keytabFileGroupAccess = "r";
+
+      // Add the relevant principal name and keytab file data to the command params state
+      commandParameters.put("principal_name", principal);
+      commandParameters.put("keytab_file", keytabFilePath);
 
-              throw e;
+      try {
+        // Get a list KERBEROS/KERBEROS_CLIENT ServiceComponentHost objects
+        List<ServiceComponentHost> serviceComponentHosts = cluster.getServiceComponentHosts(Service.Type.KERBEROS.name(), Role.KERBEROS_CLIENT.name());
+
+        if ((serviceComponentHosts != null) && !serviceComponentHosts.isEmpty()) {
+          kerberosIdentityDataFileWriter = kerberosIdentityDataFileWriterFactory.createKerberosIdentityDataFileWriter(identityDataFile);
+
+          // Iterate over the KERBEROS_CLIENT service component hosts to get the service and
+          // component-level Kerberos descriptors in order to determine which principals,
+          // keytab files needed to be created or updated.
+          for (ServiceComponentHost sch : serviceComponentHosts) {
+            if (sch.getState() == State.INSTALLED) {
+              String hostname = sch.getHostName();
+
+              kerberosIdentityDataFileWriter.writeRecord(
+                  hostname,
+                  Service.Type.KERBEROS.name(),
+                  Role.KERBEROS_CLIENT.name(),
+                  principal,
+                  principalType,
+                  keytabFilePath,
+                  keytabFileOwnerName,
+                  keytabFileOwnerAccess,
+                  keytabFileGroupName,
+                  keytabFileGroupAccess,
+                  "false");
+
+              hostsWithValidKerberosClient.add(hostname);
+              serviceComponentHostsToProcess.add(sch);
             }
           }
+        }
+
+      } catch (IOException e) {
+        String message = String.format("Failed to write index file - %s", identityDataFile.getAbsolutePath());
+        LOG.error(message);
+        throw new AmbariException(message, e);
+      } catch (Exception e) {
+        // make sure to log what is going wrong
+        LOG.error("Failed " + e);
+        throw e;
+      } finally {
+        if (kerberosIdentityDataFileWriter != null) {
+          // Make sure the data file is closed
+          try {
+            kerberosIdentityDataFileWriter.close();
+          } catch (IOException e) {
+            LOG.warn("Failed to close the index file writer", e);
+          }
+        }
+      }
 
-          // Always set up the necessary stages to perform the tasks needed to complete the operation.
-          // Some stages may be no-ops, this is expected.
-          // Gather data needed to create stages and tasks...
-          Map<String, Set<String>> clusterHostInfo = StageUtils.getClusterHostInfo(cluster);
-          String clusterHostInfoJson = StageUtils.getGson().toJson(clusterHostInfo);
-          Map<String, String> hostParams = customCommandExecutionHelper.createDefaultHostParams(cluster);
-          String hostParamsJson = StageUtils.getGson().toJson(hostParams);
-          String ambariServerHostname = StageUtils.getHostName();
-          ServiceComponentHostServerActionEvent event = new ServiceComponentHostServerActionEvent(
-              "AMBARI_SERVER",
-              ambariServerHostname, // TODO: Choose a random hostname from the cluster. All tasks for the AMBARI_SERVER service will be executed on this Ambari server
-              System.currentTimeMillis());
-          RoleCommandOrder roleCommandOrder = ambariManagementController.getRoleCommandOrder(cluster);
-
-          // If a RequestStageContainer does not already exist, create a new one...
-          if (requestStageContainer == null) {
-            requestStageContainer = new RequestStageContainer(
-                actionManager.getNextRequestId(),
-                null,
-                requestFactory,
-                actionManager);
+      // If there are ServiceComponentHosts to process, make sure the administrator credential
+      // are available
+      if (!serviceComponentHostsToProcess.isEmpty()) {
+        try {
+          validateKDCCredentials(kerberosDetails, cluster);
+        } catch (Exception e) {
+          LOG.error("Cannot validate credentials: " + e);
+          try {
+            FileUtils.deleteDirectory(dataDirectory);
+          } catch (Throwable t) {
+            LOG.warn(String.format("The data directory (%s) was not deleted due to an error condition - {%s}",
+                dataDirectory.getAbsolutePath(), t.getMessage()), t);
           }
 
-          // Use the handler implementation to setup the relevant stages.
-          // Set the service/component filter to an empty map since the service/component processing
-          // was done above.
-          handler.createStages(cluster,
-              clusterHostInfoJson, hostParamsJson, event, roleCommandOrder, kerberosDetails,
-              dataDirectory, requestStageContainer, serviceComponentHostsToProcess,
-              Collections.<String, Collection<String>>emptyMap(), null, null, hostsWithValidKerberosClient);
+          throw e;
+        }
+      }
 
+      // Always set up the necessary stages to perform the tasks needed to complete the operation.
+      // Some stages may be no-ops, this is expected.
+      // Gather data needed to create stages and tasks...
+      Map<String, Set<String>> clusterHostInfo = StageUtils.getClusterHostInfo(cluster);
+      String clusterHostInfoJson = StageUtils.getGson().toJson(clusterHostInfo);
+      Map<String, String> hostParams = customCommandExecutionHelper.createDefaultHostParams(cluster);
+      String hostParamsJson = StageUtils.getGson().toJson(hostParams);
+      String ambariServerHostname = StageUtils.getHostName();
+      ServiceComponentHostServerActionEvent event = new ServiceComponentHostServerActionEvent(
+          "AMBARI_SERVER",
+          ambariServerHostname, // TODO: Choose a random hostname from the cluster. All tasks for the AMBARI_SERVER service will be executed on this Ambari server
+          System.currentTimeMillis());
+      RoleCommandOrder roleCommandOrder = ambariManagementController.getRoleCommandOrder(cluster);
 
-          handler.addFinalizeOperationStage(cluster, clusterHostInfoJson, hostParamsJson, event,
-              dataDirectory, roleCommandOrder, requestStageContainer);
-        }
+      // If a RequestStageContainer does not already exist, create a new one...
+      if (requestStageContainer == null) {
+        requestStageContainer = new RequestStageContainer(
+            actionManager.getNextRequestId(),
+            null,
+            requestFactory,
+            actionManager);
       }
+
+      // Use the handler implementation to setup the relevant stages.
+      // Set the service/component filter to an empty map since the service/component processing
+      // was done above.
+      handler.createStages(cluster,
+          clusterHostInfoJson, hostParamsJson, event, roleCommandOrder, kerberosDetails,
+          dataDirectory, requestStageContainer, serviceComponentHostsToProcess,
+          Collections.<String, Collection<String>>emptyMap(), null, null, hostsWithValidKerberosClient);
+
+
+      handler.addFinalizeOperationStage(cluster, clusterHostInfoJson, hostParamsJson, event,
+          dataDirectory, roleCommandOrder, requestStageContainer);
     }
 
     return requestStageContainer;
@@ -2449,6 +2385,9 @@ public class KerberosHelperImpl implements KerberosHelper {
     // Add the current cluster's name
     generalProperties.put("cluster_name", cluster.getClusterName());
 
+    // Add the current date in short format (MMddyy)
+    generalProperties.put("short_date", new SimpleDateFormat("MMddyy").format(new Date()));
+
     // add clusterHostInfo config
     if (configurations.get("clusterHostInfo") == null) {
       Map<String, Set<String>> clusterHostInfo = StageUtils.getClusterHostInfo(cluster);
@@ -2702,22 +2641,22 @@ public class KerberosHelperImpl implements KerberosHelper {
     }
 
     public void addConfigureAmbariIdentityStage(Cluster cluster, String clusterHostInfoJson,
-                                          String hostParamsJson, ServiceComponentHostServerActionEvent event,
-                                          Map<String, String> commandParameters,
-                                          RoleCommandOrder roleCommandOrder, RequestStageContainer requestStageContainer)
-      throws AmbariException {
+                                                String hostParamsJson, ServiceComponentHostServerActionEvent event,
+                                                Map<String, String> commandParameters,
+                                                RoleCommandOrder roleCommandOrder, RequestStageContainer requestStageContainer)
+        throws AmbariException {
       Stage stage = createServerActionStage(requestStageContainer.getLastStageId(),
-        cluster,
-        requestStageContainer.getId(),
-        "Configure Ambari Identity",
-        clusterHostInfoJson,
-        "{}",
-        hostParamsJson,
-        ConfigureAmbariIndetityServerAction.class,
-        event,
-        commandParameters,
-        "Configure Ambari Identity",
-        configuration.getDefaultServerTaskTimeout());
+          cluster,
+          requestStageContainer.getId(),
+          "Configure Ambari Identity",
+          clusterHostInfoJson,
+          "{}",
+          hostParamsJson,
+          ConfigureAmbariIndetityServerAction.class,
+          event,
+          commandParameters,
+          "Configure Ambari Identity",
+          configuration.getDefaultServerTaskTimeout());
 
       RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder);
       roleGraph.build(stage);
@@ -3025,18 +2964,18 @@ public class KerberosHelperImpl implements KerberosHelper {
         // *****************************************************************
         // Create stage to create principals
         addCreatePrincipalsStage(cluster, clusterHostInfoJson, hostParamsJson, event, commandParameters,
-          roleCommandOrder, requestStageContainer);
+            roleCommandOrder, requestStageContainer);
 
         // *****************************************************************
         // Create stage to generate keytabs
         addCreateKeytabFilesStage(cluster, clusterHostInfoJson, hostParamsJson, event, commandParameters,
-          roleCommandOrder, requestStageContainer);
+            roleCommandOrder, requestStageContainer);
 
         // *****************************************************************
         // Create stage to distribute and configure keytab for Ambari server and configure JAAS
         if (kerberosDetails.createAmbariPrincipal()) {
           addConfigureAmbariIdentityStage(cluster, clusterHostInfoJson, hostParamsJson, event, commandParameters,
-            roleCommandOrder, requestStageContainer);
+              roleCommandOrder, requestStageContainer);
         }
 
         // *****************************************************************
@@ -3206,8 +3145,8 @@ public class KerberosHelperImpl implements KerberosHelper {
      *                             (<code>true</code>) or ignore any potential configuration changes
      *                             (<code>false</code>)
      */
-    public CreatePrincipalsAndKeytabsHandler(boolean regenerateAllKeytabs, boolean updateConfigurations, boolean
-      includeAmbariIdentity) {
+    public CreatePrincipalsAndKeytabsHandler(boolean regenerateAllKeytabs, boolean updateConfigurations,
+                                             boolean includeAmbariIdentity) {
       this.regenerateAllKeytabs = regenerateAllKeytabs;
       this.updateConfigurations = updateConfigurations;
       this.includeAmbariIdentity = includeAmbariIdentity;
@@ -3306,7 +3245,7 @@ public class KerberosHelperImpl implements KerberosHelper {
         // Create stage to distribute and configure keytab for Ambari server and configure JAAS
         if (includeAmbariIdentity && kerberosDetails.createAmbariPrincipal()) {
           addConfigureAmbariIdentityStage(cluster, clusterHostInfoJson, hostParamsJson, event, commandParameters,
-            roleCommandOrder, requestStageContainer);
+              roleCommandOrder, requestStageContainer);
         }
 
         // *****************************************************************
@@ -3477,7 +3416,7 @@ public class KerberosHelperImpl implements KerberosHelper {
     public boolean createAmbariPrincipal() {
       if (createAmbariPrincipal == null) {
         return (kerberosEnvProperties == null) ||
-          !"false".equalsIgnoreCase(kerberosEnvProperties.get(CREATE_AMBARI_PRINCIPAL));
+            !"false".equalsIgnoreCase(kerberosEnvProperties.get(CREATE_AMBARI_PRINCIPAL));
       } else {
         return createAmbariPrincipal;
       }

ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ADKerberosOperationHandler.java

@@ -235,6 +235,7 @@ public class ADKerberosOperationHandler extends KerberosOperationHandler {
    * @param password  a String containing the password to use when creating the principal
    * @param service   a boolean value indicating whether the principal is to be created as a service principal or not
    * @return an Integer declaring the generated key number
+   * @throws KerberosPrincipalAlreadyExistsException if the principal already exists
    * @throws KerberosOperationException
    */
   @Override
@@ -243,13 +244,15 @@ public class ADKerberosOperationHandler extends KerberosOperationHandler {
     if (!isOpen()) {
       throw new KerberosOperationException("This operation handler has not been opened");
     }
-
     if (principal == null) {
       throw new KerberosOperationException("principal is null");
     }
     if (password == null) {
       throw new KerberosOperationException("principal password is null");
     }
+    if (principalExists(principal)) {
+      throw new KerberosPrincipalAlreadyExistsException(principal);
+    }
 
     DeconstructedPrincipal deconstructedPrincipal = createDeconstructPrincipal(principal);
 
@@ -335,6 +338,7 @@ public class ADKerberosOperationHandler extends KerberosOperationHandler {
    * @param principal a String containing the principal to update
    * @param password  a String containing the password to set
    * @return an Integer declaring the new key number
+   * @throws KerberosPrincipalDoesNotExistException if the principal does not exist
    * @throws KerberosOperationException
    */
   @Override
@@ -348,6 +352,9 @@ public class ADKerberosOperationHandler extends KerberosOperationHandler {
     if (password == null) {
       throw new KerberosOperationException("principal password is null");
     }
+    if(!principalExists(principal)) {
+      throw new KerberosPrincipalDoesNotExistException(principal);
+    }
 
     DeconstructedPrincipal deconstructPrincipal = createDeconstructPrincipal(principal);
 

ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java

@@ -33,7 +33,9 @@ import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 
 /**
@@ -65,6 +67,12 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
   @Inject
   private SecurePasswordHelper securePasswordHelper;
 
+  /**
+   * A set of visited principal names used to prevent unnecessary processing on already processed
+   * principal names
+   */
+  private Set<String> seenPrincipals = new HashSet<String>();
+
   /**
    * Called to execute this action.  Upon invocation, calls
    * {@link org.apache.ambari.server.serveraction.kerberos.KerberosServerAction#processIdentities(java.util.Map)}
@@ -115,44 +123,50 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
       throws AmbariException {
     CommandReport commandReport = null;
 
-    boolean processPrincipal;
-    boolean regenerateKeytabs = "true".equalsIgnoreCase(getCommandParameterValue(getCommandParameters(), REGENERATE_ALL));
+    //  Only process this principal name if we haven't already processed it
+    if (!seenPrincipals.contains(evaluatedPrincipal)) {
+      seenPrincipals.add(evaluatedPrincipal);
 
-    if (regenerateKeytabs) {
-      processPrincipal = true;
-    } else {
-      KerberosPrincipalEntity kerberosPrincipalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal);
+      boolean processPrincipal;
+      boolean regenerateKeytabs = "true".equalsIgnoreCase(getCommandParameterValue(getCommandParameters(), REGENERATE_ALL));
 
-      if (kerberosPrincipalEntity == null) {
-        // This principal has not been processed before, process it.
+      if (regenerateKeytabs) {
         processPrincipal = true;
-      } else if (!StringUtils.isEmpty(kerberosPrincipalEntity.getCachedKeytabPath())) {
-        // This principal has been processed and a keytab file has been cached for it... do not process it.
-        processPrincipal = false;
-      } else if (kerberosPrincipalHostDAO.exists(evaluatedPrincipal)) {
-        // This principal has been processed and a keytab file has been distributed... do not process it.
-        processPrincipal = false;
       } else {
-        // This principal has been processed but a keytab file for it has been distributed... process it.
-        processPrincipal = true;
+        KerberosPrincipalEntity kerberosPrincipalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal);
+
+        if (kerberosPrincipalEntity == null) {
+          // This principal has not been processed before, process it.
+          processPrincipal = true;
+        } else if (!StringUtils.isEmpty(kerberosPrincipalEntity.getCachedKeytabPath())) {
+          // This principal has been processed and a keytab file has been cached for it... do not process it.
+          processPrincipal = false;
+        } else if (kerberosPrincipalHostDAO.exists(evaluatedPrincipal)) {
+          // This principal has been processed and a keytab file has been distributed... do not process it.
+          processPrincipal = false;
+        } else {
+          // This principal has been processed but a keytab file for it has been distributed... process it.
+          processPrincipal = true;
+        }
       }
-    }
 
-    if (processPrincipal) {
-      Map<String, String> principalPasswordMap = getPrincipalPasswordMap(requestSharedDataContext);
-      Map<String, Integer> principalKeyNumberMap = getPrincipalKeyNumberMap(requestSharedDataContext);
+      if (processPrincipal) {
+        Map<String, String> principalPasswordMap = getPrincipalPasswordMap(requestSharedDataContext);
 
-      String password = principalPasswordMap.get(evaluatedPrincipal);
+        String password = principalPasswordMap.get(evaluatedPrincipal);
 
-      if (password == null) {
-        boolean servicePrincipal = "service".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.PRINCIPAL_TYPE));
-        CreatePrincipalResult result = createPrincipal(evaluatedPrincipal, servicePrincipal, kerberosConfiguration, operationHandler, actionLog);
+        if (password == null) {
+          boolean servicePrincipal = "service".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.PRINCIPAL_TYPE));
+          CreatePrincipalResult result = createPrincipal(evaluatedPrincipal, servicePrincipal, kerberosConfiguration, operationHandler, regenerateKeytabs, actionLog);
 
-        if (result == null) {
-          commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
-        } else {
-          principalPasswordMap.put(evaluatedPrincipal, result.getPassword());
-          principalKeyNumberMap.put(evaluatedPrincipal, result.getKeyNumber());
+          if (result == null) {
+            commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
+          } else {
+            Map<String, Integer> principalKeyNumberMap = getPrincipalKeyNumberMap(requestSharedDataContext);
+
+            principalPasswordMap.put(evaluatedPrincipal, result.getPassword());
+            principalKeyNumberMap.put(evaluatedPrincipal, result.getKeyNumber());
+          }
         }
       }
     }
@@ -168,25 +182,25 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
    *                                 principal is a user principal
    * @param kerberosConfiguration    the kerberos-env configuration properties
    * @param kerberosOperationHandler the KerberosOperationHandler for the relevant KDC
-   * @param actionLog                the logger (may be null if no logging is desired)
-   * @return a CreatePrincipalResult containing the generated password and key number value
+   * @param regenerateKeytabs        true if this was triggered in response to regenerating keytab files; false otherwise
+   * @param actionLog                the logger (may be null if no logging is desired)  @return a CreatePrincipalResult containing the generated password and key number value
    */
   public CreatePrincipalResult createPrincipal(String principal, boolean isServicePrincipal,
                                                Map<String, String> kerberosConfiguration,
                                                KerberosOperationHandler kerberosOperationHandler,
-                                               ActionLog actionLog) {
+                                               boolean regenerateKeytabs, ActionLog actionLog) {
 
     // in case this is called directly from TopologyManager there's no HostRoleCommand
     CreatePrincipalKerberosAuditEvent.CreatePrincipalKerberosAuditEventBuilder auditEventBuilder = CreatePrincipalKerberosAuditEvent.builder()
-      .withTimestamp(System.currentTimeMillis())
-      .withRequestId(getHostRoleCommand() != null ? getHostRoleCommand().getRequestId() : -1)
-      .withTaskId(getHostRoleCommand() != null ? getHostRoleCommand().getTaskId() : -1)
-      .withPrincipal(principal);
+        .withTimestamp(System.currentTimeMillis())
+        .withRequestId(getHostRoleCommand() != null ? getHostRoleCommand().getRequestId() : -1)
+        .withTaskId(getHostRoleCommand() != null ? getHostRoleCommand().getTaskId() : -1)
+        .withPrincipal(principal);
     CreatePrincipalResult result = null;
     String message = null;
-    try {
 
-      message = String.format("Creating principal, %s", principal);
+    try {
+      message = String.format("Processing principal, %s", principal);
       LOG.info(message);
       if (actionLog != null) {
         actionLog.writeStdOut(message);
@@ -218,45 +232,60 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
       String password = securePasswordHelper.createSecurePassword(length, minLowercaseLetters, minUppercaseLetters, minDigits, minPunctuation, minWhitespace);
 
       try {
+        /**
+         * true indicates a new principal was created, false indicates an existing principal was updated
+         */
+        boolean created;
+        Integer keyNumber;
+
+        if (regenerateKeytabs) {
+          try {
+            keyNumber = kerberosOperationHandler.setPrincipalPassword(principal, password);
+            created = false;
+          } catch (KerberosPrincipalDoesNotExistException e) {
+            message = String.format("Principal, %s, does not exist, creating new principal", principal);
+            LOG.warn(message);
+            if (actionLog != null) {
+              actionLog.writeStdOut(message);
+            }
 
-        if (kerberosOperationHandler.principalExists(principal)) {
-          // Create a new password since we need to know what it is.
-          // A new password/key would have been generated after exporting the keytab anyways.
-          message = String.format("Principal, %s, already exists, setting new password", principal);
-          LOG.warn(message);
-          if (actionLog != null) {
-            actionLog.writeStdOut(message);
+            keyNumber = kerberosOperationHandler.createPrincipal(principal, password, isServicePrincipal);
+            created = true;
           }
-
-          Integer keyNumber = kerberosOperationHandler.setPrincipalPassword(principal, password);
-
-          if (keyNumber != null) {
-            result = new CreatePrincipalResult(principal, password, keyNumber);
-            message = String.format("Successfully set password for %s", principal);
-            LOG.debug(message);
-          } else {
-            message = String.format("Failed to set password for %s - unknown reason", principal);
-            LOG.error(message);
+        } else {
+          try {
+            keyNumber = kerberosOperationHandler.createPrincipal(principal, password, isServicePrincipal);
+            created = true;
+          } catch (KerberosPrincipalAlreadyExistsException e) {
+            message = String.format("Principal, %s, already exists, setting new password", principal);
+            LOG.warn(message);
             if (actionLog != null) {
-              actionLog.writeStdErr(message);
+              actionLog.writeStdOut(message);
             }
+
+            keyNumber = kerberosOperationHandler.setPrincipalPassword(principal, password);
+            created = false;
           }
-        } else {
-          message = String.format("Creating new principal, %s", principal);
-          LOG.debug(message);
+        }
 
-          Integer keyNumber = kerberosOperationHandler.createPrincipal(principal, password, isServicePrincipal);
+        if (keyNumber != null) {
+          result = new CreatePrincipalResult(principal, password, keyNumber);
 
-          if (keyNumber != null) {
-            result = new CreatePrincipalResult(principal, password, keyNumber);
+          if (created) {
             message = String.format("Successfully created new principal, %s", principal);
-            LOG.debug(message);
           } else {
+            message = String.format("Successfully set password for %s", principal);
+          }
+          LOG.debug(message);
+        } else {
+          if (created) {
             message = String.format("Failed to create principal, %s - unknown reason", principal);
-            LOG.error(message);
-            if (actionLog != null) {
-              actionLog.writeStdErr(message);
-            }
+          } else {
+            message = String.format("Failed to set password for %s - unknown reason", principal);
+          }
+          LOG.error(message);
+          if (actionLog != null) {
+            actionLog.writeStdErr(message);
           }
         }
 
@@ -272,7 +301,7 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
         }
       }
     } finally {
-      if(result == null) {
+      if (result == null) {
         auditEventBuilder.withReasonOfFailure(message == null ? "Unknown error" : message);
       }
       auditLog(auditEventBuilder.build());

ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/DestroyPrincipalsServerAction.java

@@ -29,7 +29,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 
 /**
@@ -47,6 +49,12 @@ public class DestroyPrincipalsServerAction extends KerberosServerAction {
   @Inject
   private KerberosPrincipalDAO kerberosPrincipalDAO;
 
+  /**
+   * A set of visited principal names used to prevent unnecessary processing on already processed
+   * principal names
+   */
+  private Set<String> seenPrincipals = new HashSet<String>();
+
   /**
    * Called to execute this action.  Upon invocation, calls
    * {@link KerberosServerAction#processIdentities(java.util.Map)}
@@ -88,60 +96,64 @@ public class DestroyPrincipalsServerAction extends KerberosServerAction {
                                           Map<String, Object> requestSharedDataContext)
       throws AmbariException {
 
-    String message = String.format("Destroying identity, %s", evaluatedPrincipal);
-    LOG.info(message);
-    actionLog.writeStdOut(message);
-    DestroyPrincipalKerberosAuditEvent.DestroyPrincipalKerberosAuditEventBuilder auditEventBuilder = DestroyPrincipalKerberosAuditEvent.builder()
-      .withTimestamp(System.currentTimeMillis())
-      .withRequestId(getHostRoleCommand().getRequestId())
-      .withTaskId(getHostRoleCommand().getTaskId())
-      .withPrincipal(evaluatedPrincipal);
+    // Only process this principal if we haven't already processed it
+    if (!seenPrincipals.contains(evaluatedPrincipal)) {
+      seenPrincipals.add(evaluatedPrincipal);
 
-    try {
+      String message = String.format("Destroying identity, %s", evaluatedPrincipal);
+      LOG.info(message);
+      actionLog.writeStdOut(message);
+      DestroyPrincipalKerberosAuditEvent.DestroyPrincipalKerberosAuditEventBuilder auditEventBuilder = DestroyPrincipalKerberosAuditEvent.builder()
+          .withTimestamp(System.currentTimeMillis())
+          .withRequestId(getHostRoleCommand().getRequestId())
+          .withTaskId(getHostRoleCommand().getTaskId())
+          .withPrincipal(evaluatedPrincipal);
 
       try {
-        operationHandler.removePrincipal(evaluatedPrincipal);
-      } catch (KerberosOperationException e) {
-        message = String.format("Failed to remove identity for %s from the KDC - %s", evaluatedPrincipal, e.getMessage());
-        LOG.warn(message);
-        actionLog.writeStdErr(message);
-        auditEventBuilder.withReasonOfFailure(message);
-      }
+        try {
+          operationHandler.removePrincipal(evaluatedPrincipal);
+        } catch (KerberosOperationException e) {
+          message = String.format("Failed to remove identity for %s from the KDC - %s", evaluatedPrincipal, e.getMessage());
+          LOG.warn(message);
+          actionLog.writeStdErr(message);
+          auditEventBuilder.withReasonOfFailure(message);
+        }
 
-      try {
-        KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal);
+        try {
+          KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal);
 
-        if (principalEntity != null) {
-          String cachedKeytabPath = principalEntity.getCachedKeytabPath();
+          if (principalEntity != null) {
+            String cachedKeytabPath = principalEntity.getCachedKeytabPath();
 
-          kerberosPrincipalDAO.remove(principalEntity);
+            kerberosPrincipalDAO.remove(principalEntity);
 
-        // If a cached  keytabs file exists for this principal, delete it.
-        if (cachedKeytabPath != null) {
-          if (!new File(cachedKeytabPath).delete()) {
-            LOG.debug(String.format("Failed to remove cached keytab for %s", evaluatedPrincipal));
+            // If a cached  keytabs file exists for this principal, delete it.
+            if (cachedKeytabPath != null) {
+              if (!new File(cachedKeytabPath).delete()) {
+                LOG.debug(String.format("Failed to remove cached keytab for %s", evaluatedPrincipal));
+              }
+            }
           }
-        }
-      }
 
-      // delete Ambari server keytab
-      String hostName = identityRecord.get(KerberosIdentityDataFileReader.HOSTNAME);
-      if (hostName != null && hostName.equalsIgnoreCase(KerberosHelper.AMBARI_SERVER_HOST_NAME)) {
-        String keytabFilePath = identityRecord.get(KerberosIdentityDataFileReader.KEYTAB_FILE_PATH);
-        if (keytabFilePath != null) {
-          if (!new File(keytabFilePath).delete()) {
-            LOG.debug(String.format("Failed to remove ambari keytab for %s", evaluatedPrincipal));
+          // delete Ambari server keytab
+          String hostName = identityRecord.get(KerberosIdentityDataFileReader.HOSTNAME);
+          if (hostName != null && hostName.equalsIgnoreCase(KerberosHelper.AMBARI_SERVER_HOST_NAME)) {
+            String keytabFilePath = identityRecord.get(KerberosIdentityDataFileReader.KEYTAB_FILE_PATH);
+            if (keytabFilePath != null) {
+              if (!new File(keytabFilePath).delete()) {
+                LOG.debug(String.format("Failed to remove ambari keytab for %s", evaluatedPrincipal));
+              }
+            }
           }
+        } catch (Throwable t) {
+          message = String.format("Failed to remove identity for %s from the Ambari database - %s", evaluatedPrincipal, t.getMessage());
+          LOG.warn(message);
+          actionLog.writeStdErr(message);
+          auditEventBuilder.withReasonOfFailure(message);
         }
+      } finally {
+        auditLog(auditEventBuilder.build());
       }
-    } catch (Throwable t) {
-        message = String.format("Failed to remove identity for %s from the Ambari database - %s", evaluatedPrincipal, t.getMessage());
-        LOG.warn(message);
-        actionLog.writeStdErr(message);
-        auditEventBuilder.withReasonOfFailure(message);
-      }
-    } finally {
-      auditLog(auditEventBuilder.build());
     }
 
     // There is no reason to fail this task if an identity was not removed. The cluster will work

ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosMissingAdminCredentialsException.java

@@ -22,6 +22,25 @@ package org.apache.ambari.server.serveraction.kerberos;
  * Indicates that the KDC admin credentials have not been set.
  */
 public class KerberosMissingAdminCredentialsException extends KerberosOperationException {
+  /**
+   * The default error message to use when handling this exception
+   */
+  private static final String DEFAULT_MESSAGE = "Missing KDC administrator credentials.\n" +
+      "The KDC administrator credentials must be set as a persisted or temporary credential resource." +
+      "This may be done by issuing a POST to the /api/v1/clusters/:clusterName/credentials/kdc.admin.credential API entry point with the following payload:\n" +
+      "{\n" +
+      "  \"Credential\" : {\n" +
+      "    \"principal\" : \"(PRINCIPAL)\", \"key\" : \"(PASSWORD)\", \"type\" : \"(persisted|temporary)\"}\n" +
+      "  }\n" +
+      "}";
+
+  /**
+   * Constructor using the default missing credentials message.
+   */
+  public KerberosMissingAdminCredentialsException() {
+    this(DEFAULT_MESSAGE);
+  }
+
   /**
    * Constructor.
    *
@@ -31,13 +50,22 @@ public class KerberosMissingAdminCredentialsException extends KerberosOperationE
     super(message);
   }
 
+  /**
+   * Constructor using the default message.
+   *
+   * @param cause   root cause
+   */
+  public KerberosMissingAdminCredentialsException(Throwable cause) {
+    this(DEFAULT_MESSAGE, cause);
+  }
+
   /**
    * Constructor.
    *
-   * @param message  error message
-   * @param cause    root cause
+   * @param message error message
+   * @param cause   root cause
    */
   public KerberosMissingAdminCredentialsException(String message, Throwable cause) {
     super(message, cause);
   }
-}
+}

ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosOperationHandler.java

@@ -266,6 +266,7 @@ public abstract class KerberosOperationHandler {
    * @param service   a boolean value indicating whether the principal is to be created as a service principal or not
    * @return an Integer declaring the generated key number
    * @throws KerberosOperationException
+   * @throws KerberosPrincipalAlreadyExistsException if the principal already exists
    */
   public abstract Integer createPrincipal(String principal, String password, boolean service)
       throws KerberosOperationException;
@@ -279,6 +280,7 @@ public abstract class KerberosOperationHandler {
    * @param password  a String containing the password to set
    * @return an Integer declaring the new key number
    * @throws KerberosOperationException
+   * @throws KerberosPrincipalDoesNotExistException if the principal does not exist
    */
   public abstract Integer setPrincipalPassword(String principal, String password)
       throws KerberosOperationException;

ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosPrincipalAlreadyExistsException.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.serveraction.kerberos;
+
+/**
+ * KerberosPrincipalAlreadyExistsException is a KerberosOperationException thrown in the event a
+ * request to create a new princip7als was made but the princial already exists in the KDC.
+ */
+public class KerberosPrincipalAlreadyExistsException extends KerberosOperationException {
+
+  /**
+   * Creates a new KerberosPrincipalAlreadyExistsException with a message
+   *
+   * @param message a String containing the message indicating the reason for this exception
+   */
+  public KerberosPrincipalAlreadyExistsException(String message) {
+    super(message);
+  }
+
+  /**
+   * Creates a new KerberosPrincipalAlreadyExistsException with a message and a cause
+   *
+   * @param message a String containing the message indicating the reason for this exception
+   * @param cause   a Throwable declaring the previously thrown Throwable that led to this exception
+   */
+  public KerberosPrincipalAlreadyExistsException(String message, Throwable cause) {
+    super(message, cause);
+  }
+}

ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosPrincipalDoesNotExistException.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.serveraction.kerberos;
+
+/**
+ * KerberosPrincipalDoesNotExistException is a KerberosOperationException thrown in the event a
+ * request to modify an existing principal was made but the princial does not exist in the KDC.
+ */
+public class KerberosPrincipalDoesNotExistException extends KerberosOperationException {
+
+  /**
+   * Creates a new KerberosPrincipalDoesNotExistException with a message
+   *
+   * @param message a String containing the message indicating the reason for this exception
+   */
+  public KerberosPrincipalDoesNotExistException(String message) {
+    super(message);
+  }
+
+  /**
+   * Creates a new KerberosPrincipalDoesNotExistException with a message and a cause
+   *
+   * @param message a String containing the message indicating the reason for this exception
+   * @param cause   a Throwable declaring the previously thrown Throwable that led to this exception
+   */
+  public KerberosPrincipalDoesNotExistException(String message, Throwable cause) {
+    super(message, cause);
+  }
+}

ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandler.java

@@ -165,10 +165,11 @@ public class MITKerberosOperationHandler extends KerberosOperationHandler {
    * @param password  a String containing the password to use when creating the principal
    * @param service   a boolean value indicating whether the principal is to be created as a service principal or not
    * @return an Integer declaring the generated key number
-   * @throws KerberosKDCConnectionException       if a connection to the KDC cannot be made
-   * @throws KerberosAdminAuthenticationException if the administrator credentials fail to authenticate
-   * @throws KerberosRealmException               if the realm does not map to a KDC
-   * @throws KerberosOperationException           if an unexpected error occurred
+   * @throws KerberosKDCConnectionException          if a connection to the KDC cannot be made
+   * @throws KerberosAdminAuthenticationException    if the administrator credentials fail to authenticate
+   * @throws KerberosRealmException                  if the realm does not map to a KDC
+   * @throws KerberosPrincipalAlreadyExistsException if the principal already exists
+   * @throws KerberosOperationException              if an unexpected error occurred
    */
   @Override
   public Integer createPrincipal(String principal, String password, boolean service)
@@ -193,6 +194,8 @@ public class MITKerberosOperationHandler extends KerberosOperationHandler {
       String stdOut = result.getStdout();
       if ((stdOut != null) && stdOut.contains(String.format("Principal \"%s\" created", principal))) {
         return getKeyNumber(principal);
+      } else if ((stdOut != null) && stdOut.contains(String.format("Principal or policy already exists while creating \"%s\"", principal))) {
+        throw new KerberosPrincipalAlreadyExistsException(principal);
       } else {
         LOG.error("Failed to execute kadmin query: add_principal -pw \"********\" {} {}\nSTDOUT: {}\nSTDERR: {}",
             (createAttributes == null) ? "" : createAttributes, principal, stdOut, result.getStderr());
@@ -211,10 +214,11 @@ public class MITKerberosOperationHandler extends KerberosOperationHandler {
    * @param principal a String containing the principal to update
    * @param password  a String containing the password to set
    * @return an Integer declaring the new key number
-   * @throws KerberosKDCConnectionException       if a connection to the KDC cannot be made
-   * @throws KerberosAdminAuthenticationException if the administrator credentials fail to authenticate
-   * @throws KerberosRealmException               if the realm does not map to a KDC
-   * @throws KerberosOperationException           if an unexpected error occurred
+   * @throws KerberosKDCConnectionException         if a connection to the KDC cannot be made
+   * @throws KerberosAdminAuthenticationException   if the administrator credentials fail to authenticate
+   * @throws KerberosRealmException                 if the realm does not map to a KDC
+   * @throws KerberosPrincipalDoesNotExistException if the principal does not exist
+   * @throws KerberosOperationException             if an unexpected error occurred
    */
   @Override
   public Integer setPrincipalPassword(String principal, String password) throws KerberosOperationException {
@@ -228,9 +232,19 @@ public class MITKerberosOperationHandler extends KerberosOperationHandler {
       throw new KerberosOperationException("Failed to set password - no password specified");
     } else {
       // Create the kdamin query:  change_password <-randkey|-pw <password>> <principal>
-      invokeKAdmin(String.format("change_password -pw \"%s\" %s", password, principal));
+      ShellCommandUtil.Result result = invokeKAdmin(String.format("change_password -pw \"%s\" %s", password, principal));
 
-      return getKeyNumber(principal);
+      String stdOut = result.getStdout();
+      if ((stdOut != null) && stdOut.contains(String.format("Password for \"%s\" changed", principal))) {
+        return getKeyNumber(principal);
+      } else if ((stdOut != null) && stdOut.contains("Principal does not exist")) {
+        throw new KerberosPrincipalDoesNotExistException(principal);
+      } else {
+        LOG.error("Failed to execute kadmin query: change_password -pw \"********\" {} \nSTDOUT: {}\nSTDERR: {}",
+            principal, stdOut, result.getStderr());
+        throw new KerberosOperationException(String.format("Failed to update password for %s\nSTDOUT: %s\nSTDERR: %s",
+            principal, stdOut, result.getStderr()));
+      }
     }
   }
 

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/kerberos.json

@@ -11,6 +11,9 @@
         {
           "name": "METRICS_COLLECTOR",
           "identities": [
+            {
+              "name": "/HDFS/NAMENODE/hdfs"
+            },
             {
               "name": "ams_hbase_master_hbase",
               "principal": {

ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java

@@ -2603,7 +2603,7 @@ public class KerberosHelperTest extends EasyMockSupport {
     Capture<? extends String> capturePrincipalForKeytab = newCapture(CaptureType.ALL);
 
     CreatePrincipalsServerAction createPrincipalsServerAction = injector.getInstance(CreatePrincipalsServerAction.class);
-    expect(createPrincipalsServerAction.createPrincipal(capture(capturePrincipal), eq(false), anyObject(Map.class),  anyObject(KerberosOperationHandler.class), isNull(ActionLog.class)))
+    expect(createPrincipalsServerAction.createPrincipal(capture(capturePrincipal), eq(false), anyObject(Map.class),  anyObject(KerberosOperationHandler.class), eq(false), isNull(ActionLog.class)))
         .andReturn(new CreatePrincipalsServerAction.CreatePrincipalResult("anything", "password", 1))
         .times(3);
 
@@ -3214,20 +3214,19 @@ public class KerberosHelperTest extends EasyMockSupport {
     KerberosHelper kerberosHelper = injector.getInstance(KerberosHelper.class);
     boolean managingIdentities = !Boolean.FALSE.equals(manageIdentities);
 
-    final Map<String, String> kerberosEnvProperties = createMock(Map.class);
-    expect(kerberosEnvProperties.get("kdc_type")).andReturn("mit-kdc").anyTimes();
-    expect(kerberosEnvProperties.get("realm")).andReturn("FOOBAR.COM").anyTimes();
-
-    expect(kerberosEnvProperties.get("manage_identities"))
-        .andReturn((manageIdentities == null)
+    final Map<String, String> kerberosEnvProperties = new HashMap<String,String>();
+    kerberosEnvProperties.put("kdc_type","mit-kdc");
+    kerberosEnvProperties.put("realm","FOOBAR.COM");
+    kerberosEnvProperties.put("manage_identities","FOOBAR.COM");
+    kerberosEnvProperties.put("manage_identities",
+        (manageIdentities == null)
             ? null
-            : ((manageIdentities) ? "true" : "false"))
-        .anyTimes();
+            : ((manageIdentities) ? "true" : "false"));
 
     final Config kerberosEnvConfig = createMock(Config.class);
     expect(kerberosEnvConfig.getProperties()).andReturn(kerberosEnvProperties).anyTimes();
 
-    final Map<String, String> krb5ConfProperties = createMock(Map.class);
+    final Map<String, String> krb5ConfProperties = new HashMap<String, String>();
 
     final Config krb5ConfConfig = createMock(Config.class);
     expect(krb5ConfConfig.getProperties()).andReturn(krb5ConfProperties).anyTimes();
@@ -3252,20 +3251,6 @@ public class KerberosHelperTest extends EasyMockSupport {
       expect(schKerberosClient.getHostName()).andReturn("host1").anyTimes();
       expect(schKerberosClient.getState()).andReturn(State.INSTALLED).anyTimes();
 
-      final ServiceComponentHost sch1 = createMock(ServiceComponentHost.class);
-      expect(sch1.getServiceName()).andReturn("SERVICE1").anyTimes();
-      expect(sch1.getServiceComponentName()).andReturn("COMPONENT1").anyTimes();
-      expect(sch1.getHostName()).andReturn("host1").anyTimes();
-
-      final ServiceComponentHost sch2 = createStrictMock(ServiceComponentHost.class);
-      expect(sch2.getServiceName()).andReturn("SERVICE2").anyTimes();
-      expect(sch2.getServiceComponentName()).andReturn("COMPONENT3").anyTimes();
-
-      final ServiceComponentHost sch3 = createStrictMock(ServiceComponentHost.class);
-      expect(sch3.getServiceName()).andReturn("SERVICE3").anyTimes();
-      expect(sch3.getServiceComponentName()).andReturn("COMPONENT3").anyTimes();
-      expect(sch3.getHostName()).andReturn("host1").anyTimes();
-
       final ServiceComponent serviceComponentKerberosClient = createNiceMock(ServiceComponent.class);
       expect(serviceComponentKerberosClient.getName()).andReturn(Role.KERBEROS_CLIENT.name()).anyTimes();
       expect(serviceComponentKerberosClient.getServiceComponentHosts()).andReturn(Collections.singletonMap("host1", schKerberosClient)).anyTimes();
@@ -3299,16 +3284,13 @@ public class KerberosHelperTest extends EasyMockSupport {
             }
           })
           .anyTimes();
-      expect(cluster.getServiceComponentHosts("host1"))
+      expect(cluster.getServiceComponentHosts(Service.Type.KERBEROS.name(), Role.KERBEROS_CLIENT.name()))
           .andReturn(new ArrayList<ServiceComponentHost>() {
             {
-              add(sch1);
-              add(sch2);
-              add(sch3);
               add(schKerberosClient);
             }
           })
-          .once();
+          .anyTimes();
       expect(cluster.getCurrentStackVersion())
           .andReturn(new StackId("HDP", "2.2"))
           .anyTimes();
@@ -3325,21 +3307,11 @@ public class KerberosHelperTest extends EasyMockSupport {
       }).anyTimes();
 
       final Clusters clusters = injector.getInstance(Clusters.class);
-      expect(clusters.getHostsForCluster("c1"))
-          .andReturn(new HashMap<String, Host>() {
-            {
-              put("host1", host);
-            }
-          })
-          .once();
       expect(clusters.getHost("host1"))
           .andReturn(host)
           .once();
 
       final AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
-      expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, "host1"))
-          .andReturn(Collections.<String, Map<String, String>>emptyMap())
-          .once();
       expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null))
           .andReturn(Collections.<String, Map<String, String>>emptyMap())
           .once();
@@ -3505,37 +3477,24 @@ public class KerberosHelperTest extends EasyMockSupport {
           }
         })
         .anyTimes();
-    expect(cluster.getServiceComponentHosts("host1"))
+    expect(cluster.getServiceComponentHosts(Service.Type.KERBEROS.name(), Role.KERBEROS_CLIENT.name()))
         .andReturn(new ArrayList<ServiceComponentHost>() {
           {
-            add(sch1);
-            add(sch2);
-            add(sch3);
             add(schKerberosClient);
           }
         })
-        .once();
+        .anyTimes();
     expect(cluster.getCurrentStackVersion())
         .andReturn(new StackId("HDP", "2.2"))
         .anyTimes();
     expect(cluster.getSessionAttributes()).andReturn(new HashMap<String, Object>()).anyTimes();
 
     final Clusters clusters = injector.getInstance(Clusters.class);
-    expect(clusters.getHostsForCluster("c1"))
-        .andReturn(new HashMap<String, Host>() {
-          {
-            put("host1", host);
-          }
-        })
-        .once();
     expect(clusters.getHost("host1"))
         .andReturn(host)
         .once();
 
     final AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
-    expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, "host1"))
-        .andReturn(Collections.<String, Map<String, String>>emptyMap())
-        .once();
     expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null))
         .andReturn(Collections.<String, Map<String, String>>emptyMap())
         .once();