AMBARI-14409. Blueprints Kerberos deployments fail intermittently due to invalid keytabs. (Sandor Magyari via rnettleton)

ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java

@@ -382,10 +382,20 @@ public class KerberosHelperImpl implements KerberosHelper {
 
       Map<String, String> kerberosDescriptorProperties = kerberosDescriptor.getProperties();
       Map<String, Map<String, String>> configurations = addAdditionalConfigurations(cluster,
-          deepCopy(existingConfigurations), null, kerberosDescriptorProperties);
+        deepCopy(existingConfigurations), null, kerberosDescriptorProperties);
 
       Map<String, String> kerberosConfiguration = kerberosDetails.getKerberosEnvProperties();
       KerberosOperationHandler kerberosOperationHandler = kerberosOperationHandlerFactory.getKerberosOperationHandler(kerberosDetails.getKdcType());
+      PrincipalKeyCredential administratorCredential = getKDCAdministratorCredentials(cluster.getClusterName());
+
+      try {
+        kerberosOperationHandler.open(administratorCredential, kerberosDetails.getDefaultRealm(), kerberosConfiguration);
+      } catch (KerberosOperationException e) {
+        String message = String.format("Failed to process the identities, could not properly open the KDC operation handler: %s",
+          e.getMessage());
+        LOG.error(message);
+        throw new AmbariException(message, e);
+      }
 
       for (String serviceName : services) {
         // Set properties...
@@ -416,6 +426,15 @@ public class KerberosHelperImpl implements KerberosHelper {
           }
         }
       }
+
+      // The KerberosOperationHandler needs to be closed, if it fails to close ignore the
+      // exception since there is little we can or care to do about it now.
+      try {
+        kerberosOperationHandler.close();
+      } catch (KerberosOperationException e) {
+        // Ignore this...
+      }
+
     }
 
     return true;

ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java

@@ -114,6 +114,7 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
     boolean regenerateKeytabs = "true".equalsIgnoreCase(getCommandParameterValue(getCommandParameters(), REGENERATE_ALL));
 
     if (regenerateKeytabs || !kerberosPrincipalHostDAO.exists(evaluatedPrincipal)) {
+
       Map<String, String> principalPasswordMap = getPrincipalPasswordMap(requestSharedDataContext);
       Map<String, Integer> principalKeyNumberMap = getPrincipalKeyNumberMap(requestSharedDataContext);
 
@@ -201,6 +202,7 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
         if (keyNumber != null) {
           message = String.format("Successfully set password for %s", principal);
           LOG.debug(message);
+          result = new CreatePrincipalResult(principal, password, keyNumber);
         } else {
           message = String.format("Failed to set password for %s - unknown reason", principal);
           LOG.error(message);

ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java

@@ -110,6 +110,10 @@ public class ClusterConfigurationRequest {
     Configuration clusterConfiguration = clusterTopology.getConfiguration();
 
     try {
+      AmbariContext.getController().getKerberosHelper()
+        .ensureHeadlessIdentities(cluster, clusterConfiguration.getFullProperties(),
+          new HashSet<String>(blueprint.getServices()));
+
       Map<String, Map<String, String>> updatedConfigs = AmbariContext.getController().getKerberosHelper()
         .getServiceConfigurationUpdates(cluster, clusterConfiguration.getFullProperties(),
         new HashSet<String>(blueprint.getServices()));

ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java

@@ -2401,7 +2401,7 @@ public class KerberosHelperTest extends EasyMockSupport {
     expect(cluster.getDesiredConfigByType("kerberos-env")).andReturn(configKerberosEnv).times(1);
     expect(cluster.getSecurityType()).andReturn(SecurityType.KERBEROS).times(1);
     expect(cluster.getCurrentStackVersion()).andReturn(new StackId("HDP", "2.2")).times(1);
-    expect(cluster.getClusterName()).andReturn("c1").times(2);
+    expect(cluster.getClusterName()).andReturn("c1").times(4);
     expect(cluster.getHosts()).andReturn(Arrays.asList(host1, host2, host3)).times(1);
     expect(cluster.getServices()).andReturn(servicesMap).times(1);
 
@@ -2491,6 +2491,10 @@ public class KerberosHelperTest extends EasyMockSupport {
     AmbariMetaInfo ambariMetaInfo = injector.getInstance(AmbariMetaInfo.class);
     ambariMetaInfo.init();
 
+    CredentialStoreService credentialStoreService = injector.getInstance(CredentialStoreService.class);
+    credentialStoreService.setCredential(cluster.getClusterName(), KerberosHelper.KDC_ADMINISTRATOR_CREDENTIAL_ALIAS,
+      new PrincipalKeyCredential("principal", "password"), CredentialStoreType.TEMPORARY);
+
     KerberosHelper kerberosHelper = injector.getInstance(KerberosHelper.class);
     kerberosHelper.ensureHeadlessIdentities(cluster, existingConfigurations, services);
 

ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java

@@ -125,7 +125,7 @@ public class ClusterConfigurationRequestTest {
     expectLastCall().andReturn(controller).anyTimes();
 
     expect(controller.getClusters()).andReturn(clusters).anyTimes();
-    expect(controller.getKerberosHelper()).andReturn(kerberosHelper).once();
+    expect(controller.getKerberosHelper()).andReturn(kerberosHelper).times(2);
 
     expect(clusters.getCluster("testCluster")).andReturn(cluster).anyTimes();
 
@@ -154,8 +154,11 @@ public class ClusterConfigurationRequestTest {
     Map<String, String> properties = new HashMap<>();
     properties.put("testPorperty", "testValue");
     kerberosConfig.put("testConfigType", properties);
+    expect(kerberosHelper.ensureHeadlessIdentities(anyObject(Cluster.class), anyObject(Map.class), anyObject
+      (Set.class))).andReturn(true).once();
     expect(kerberosHelper.getServiceConfigurationUpdates(anyObject(Cluster.class), anyObject(Map.class), anyObject
-      (Set.class))).andReturn(kerberosConfig).anyTimes();
+      (Set.class))).andReturn(kerberosConfig).once();
+
 
     PowerMock.replay(stack, blueprint, topology, controller, clusters, kerberosHelper, ambariContext,
       AmbariContext