AMBARI-13171 - Support Host in MM Removal During Upgrade (jonathanhurley)

ambari-server/src/main/java/org/apache/ambari/server/events/HostRemovedEvent.java

@@ -17,18 +17,43 @@
  */
 package org.apache.ambari.server.events;
 
+import java.util.Collections;
+import java.util.Set;
+
+import org.apache.ambari.server.state.Cluster;
+
 /**
  * The {@link HostRemovedEvent} class is fired when a host is removed from the
  * cluster.
  */
 public class HostRemovedEvent extends HostEvent {
+
+  /**
+   * The clusters that the removed host belonged to.
+   */
+  private final Set<Cluster> m_clusters;
+
   /**
    * Constructor.
    *
    * @param hostName
    */
-  public HostRemovedEvent(String hostName) {
+  public HostRemovedEvent(String hostName, Set<Cluster> clusters) {
     super(AmbariEventType.HOST_REMOVED, hostName);
+    m_clusters = clusters;
+  }
+
+  /**
+   * The clusters that the host belonged to.
+   *
+   * @return the clusters, or an empty set.
+   */
+  public Set<Cluster> getClusters() {
+    if (null == m_clusters) {
+      return Collections.emptySet();
+    }
+
+    return m_clusters;
   }
 
   /**

ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/HostVersionOutOfSyncListener.java

@@ -29,6 +29,7 @@ import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.EagerSingleton;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.events.HostAddedEvent;
+import org.apache.ambari.server.events.HostRemovedEvent;
 import org.apache.ambari.server.events.ServiceComponentInstalledEvent;
 import org.apache.ambari.server.events.ServiceInstalledEvent;
 import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
@@ -211,4 +212,52 @@ public class HostVersionOutOfSyncListener {
     }
   }
 
+  /**
+   * Recalculates the cluster repo version state when a host is removed. If
+   * hosts are removed during an upgrade, the remaining hosts will all be in the
+   * {@link RepositoryVersionState#INSTALLED} state, but the cluster will never
+   * transition into this state. This is because when the host is removed, a
+   * recalculation must happen.
+   *
+   * @param event
+   *          the removal event.
+   */
+  @Subscribe
+  @Transactional
+  public void onHostEvent(HostRemovedEvent event) {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug(event.toString());
+    }
+
+    try {
+      Set<Cluster> clusters = event.getClusters();
+      for (Cluster cluster : clusters) {
+        Collection<ClusterVersionEntity> allClusterVersions = cluster.getAllClusterVersions();
+
+        for (ClusterVersionEntity clusterVersion : allClusterVersions) {
+          RepositoryVersionState repositoryVersionState = clusterVersion.getState();
+
+          // the CURRENT/INSTALLED states should not be affected by a host
+          // removal - if it's already current then removing a host will never
+          // make it not CURRENT or not INSTALLED
+          switch (repositoryVersionState) {
+            case CURRENT:
+            case INSTALLED:
+              continue;
+            default:
+              break;
+          }
+
+          RepositoryVersionEntity repositoryVersion = clusterVersion.getRepositoryVersion();
+          cluster.recalculateClusterVersionState(repositoryVersion);
+        }
+      }
+
+    } catch (AmbariException ambariException) {
+      LOG.error(
+          "Unable to recalculate the cluster repository version state when a host was removed",
+          ambariException);
+    }
+  }
+
 }

ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterEntity.java

@@ -248,7 +248,7 @@ public class ClusterEntity {
 
   @Override
   public int hashCode() {
-    int result = clusterId.hashCode();
+    int result = null == clusterId ? 0 : clusterId.hashCode();
     result = 31 * result + clusterName.hashCode();
     return result;
   }

ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java

@@ -201,7 +201,7 @@ public class FinalizeUpgradeAction extends AbstractServerAction {
 
       checkHostComponentVersions(cluster, version, clusterDesiredStackId);
 
-      // May need to first transition to UPGRADED
+      // May need to first transition to UPGRADED from UPGRADING
       if (atLeastOneHostInInstalledState) {
         cluster.transitionClusterVersion(clusterDesiredStackId, version,
             RepositoryVersionState.UPGRADED);

ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java

@@ -204,7 +204,7 @@ public class MasterHostResolver {
         // !!! FIXME: only rely on maintenance state once the upgrade endpoint
         // is using the pre-req endpoint for determining if an upgrade is
         // possible
-        if (maintenanceState != MaintenanceState.OFF && !sc.isMasterComponent()) {
+        if (maintenanceState != MaintenanceState.OFF) {
           unhealthyHosts.add(sch);
         } else if (null == m_version || null == sch.getVersion() || !sch.getVersion().equals(m_version)) {
           upgradeHosts.add(hostName);

ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java

@@ -1465,8 +1465,10 @@ public class ClusterImpl implements Cluster {
       List<ClusterVersionEntity> clusterVersionEntities = clusterVersionDAO.findByCluster(getClusterName());
       StackId currentStackId = getCurrentStackVersion();
       for (ClusterVersionEntity clusterVersionEntity : clusterVersionEntities) {
-        if (clusterVersionEntity.getRepositoryVersion().getStack().equals(
-            currentStackId.getStackId())
+        RepositoryVersionEntity repositoryVersionEntity = clusterVersionEntity.getRepositoryVersion();
+        StackId repoVersionStackId = repositoryVersionEntity.getStackId();
+
+        if (repoVersionStackId.equals(currentStackId)
             && clusterVersionEntity.getState() != RepositoryVersionState.CURRENT) {
           recalculateClusterVersionState(clusterVersionEntity.getRepositoryVersion());
         }
@@ -2855,16 +2857,16 @@ public class ClusterImpl implements Cluster {
     clusterGlobalLock.writeLock().lock();
     try {
       Collection<ClusterConfigMappingEntity> configMappingEntities = clusterEntity.getConfigMappingEntities();
-      
+
       // disable previous config
       for (ClusterConfigMappingEntity e : configMappingEntities) {
         LOG.debug("{} with tag {} is unselected", e.getType(), e.getTag());
         e.setSelected(0);
       }
-      
+
       List<ClusterConfigMappingEntity> clusterConfigMappingEntities = clusterDAO.getClusterConfigMappingsByStack(clusterEntity.getClusterId(), stackId);
       Collection<ClusterConfigMappingEntity> latestConfigMappingByStack = getLatestConfigMapping(clusterConfigMappingEntities);
-      
+
       for(ClusterConfigMappingEntity e: configMappingEntities){
         String type = e.getType(); //loop thru all the config mappings
         String tag =  e.getTag();
@@ -2877,7 +2879,7 @@ public class ClusterImpl implements Cluster {
           }
         }
       }
-      
+
       clusterEntity = clusterDAO.merge(clusterEntity);
 
       cacheConfigurations();
@@ -2891,7 +2893,7 @@ public class ClusterImpl implements Cluster {
     for (ClusterConfigMappingEntity e : clusterConfigMappingEntities) {
       String type = e.getType();
       if(temp.containsKey(type)){
-        ClusterConfigMappingEntity entityStored = (ClusterConfigMappingEntity)temp.get(type);
+        ClusterConfigMappingEntity entityStored = temp.get(type);
         Long timestampStored = entityStored.getCreateTimestamp();
         Long timestamp = e.getCreateTimestamp();
         if(timestamp > timestampStored){
@@ -2902,7 +2904,7 @@ public class ClusterImpl implements Cluster {
       }
     }
 
-    return (Collection<ClusterConfigMappingEntity>) temp.values();
+    return temp.values();
   }
 
   /**

ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java

@@ -778,14 +778,42 @@ public class ClustersImpl implements Clusters {
   }
 
   /***
-   * Delete a host entirely from the cluster and all database tables, except AlertHistory.
-   * If the host is not found, throws {@link org.apache.ambari.server.HostNotFoundException}
+   * Delete a host entirely from the cluster and all database tables, except
+   * AlertHistory. If the host is not found, throws
+   * {@link org.apache.ambari.server.HostNotFoundException}.
+   * <p/>
+   * This method will trigger a {@link HostRemovedEvent} when completed.
+   *
    * @param hostname
    * @throws AmbariException
    */
   @Override
-  @Transactional
   public void deleteHost(String hostname) throws AmbariException {
+    // unmapping hosts from a cluster modifies the collections directly; keep
+    // a copy of this to ensure that we can pass in the original set of
+    // clusters that the host belonged to to the host removal event
+    Set<Cluster> clusters = hostClusterMap.get(hostname);
+    Set<Cluster> hostsClusters = new HashSet<>(clusters);
+
+    deleteHostEntityRelationships(hostname);
+
+    // Publish the event, using the original list of clusters that the host
+    // belonged to
+    HostRemovedEvent event = new HostRemovedEvent(hostname, hostsClusters);
+    eventPublisher.publish(event);
+  }
+
+  /***
+   * Deletes all of the JPA relationships between a host and other entities.
+   * This method will not fire {@link HostRemovedEvent} since it is performed
+   * within an {@link Transactional} and the event must fire after the
+   * transaction is successfully committed.
+   *
+   * @param hostname
+   * @throws AmbariException
+   */
+  @Transactional
+  private void deleteHostEntityRelationships(String hostname) throws AmbariException {
     checkLoaded();
 
     if (!hosts.containsKey(hostname)) {
@@ -796,13 +824,14 @@ public class ClustersImpl implements Clusters {
 
     try {
       HostEntity entity = hostDAO.findByName(hostname);
-      
+
       if (entity == null) {
         return;
       }
       // Remove from all clusters in the cluster_host_mapping table.
-      // This will also remove from kerberos_principal_hosts, hostconfigmapping, and configgrouphostmapping 
+      // This will also remove from kerberos_principal_hosts, hostconfigmapping, and configgrouphostmapping
       Set<Cluster> clusters = hostClusterMap.get(hostname);
+
       unmapHostFromClusters(hostname, clusters);
       hostDAO.refresh(entity);
 
@@ -822,10 +851,6 @@ public class ClustersImpl implements Clusters {
 
       hostDAO.remove(entity);
 
-      // Publish the event
-      HostRemovedEvent event = new HostRemovedEvent(hostname);
-      eventPublisher.publish(event);
-      
       // Note, if the host is still heartbeating, then new records will be re-inserted
       // into the hosts and hoststate tables
     } catch (Exception e) {

ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/HostVersionOutOfSyncListenerTest.java

@@ -19,8 +19,6 @@
 package org.apache.ambari.server.events.listeners.upgrade;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -32,10 +30,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import com.google.inject.Inject;
-import com.google.inject.persist.UnitOfWork;
 import org.apache.ambari.server.AmbariException;
-import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.events.HostRemovedEvent;
 import org.apache.ambari.server.events.ServiceComponentInstalledEvent;
 import org.apache.ambari.server.events.ServiceInstalledEvent;
 import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
@@ -54,7 +50,6 @@ import org.apache.ambari.server.state.RepositoryVersionState;
 import org.apache.ambari.server.state.Service;
 import org.apache.ambari.server.state.ServiceComponent;
 import org.apache.ambari.server.state.ServiceComponentHostFactory;
-import org.apache.ambari.server.state.ServiceFactory;
 import org.apache.ambari.server.state.StackId;
 import org.apache.ambari.server.utils.EventBusSynchronizer;
 import org.junit.After;
@@ -64,8 +59,10 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.inject.Guice;
+import com.google.inject.Inject;
 import com.google.inject.Injector;
 import com.google.inject.persist.PersistService;
+import com.google.inject.persist.UnitOfWork;
 
 public class HostVersionOutOfSyncListenerTest {
   private static final Logger LOG = LoggerFactory.getLogger(HostVersionOutOfSyncListenerTest.class);
@@ -84,15 +81,9 @@ public class HostVersionOutOfSyncListenerTest {
   @Inject
   private HostVersionDAO hostVersionDAO;
 
-  @Inject
-  private AmbariMetaInfo metaInfo;
-
   @Inject
   private ServiceComponentHostFactory serviceComponentHostFactory;
 
-  @Inject
-  private ServiceFactory serviceFactory;
-
   @Inject
   private AmbariEventPublisher m_eventPublisher;
 
@@ -104,7 +95,6 @@ public class HostVersionOutOfSyncListenerTest {
 
     EventBusSynchronizer.synchronizeAmbariEventPublisher(injector);
     injector.injectMembers(this);
-    injector.getInstance(UnitOfWork.class).begin();
 
     StackId stackId = new StackId(this.stackId);
     clusters.addCluster("c1", stackId);
@@ -119,7 +109,6 @@ public class HostVersionOutOfSyncListenerTest {
 
   @After
   public void teardown() {
-    injector.getInstance(UnitOfWork.class).end();
     injector.getInstance(PersistService.class).stop();
   }
 
@@ -145,9 +134,10 @@ public class HostVersionOutOfSyncListenerTest {
     c1.createClusterVersion(stackId, INSTALLED_VERSION, "admin", RepositoryVersionState.INSTALLING);
     c1.setCurrentStackVersion(stackId);
     c1.recalculateAllClusterVersionStates();
-    checkStackVersionState(stackId.getStackId(), INSTALLED_VERSION, RepositoryVersionState.INSTALLING);
+    assertRepoVersionState(stackId.getStackId(), INSTALLED_VERSION,
+        RepositoryVersionState.INSTALLING);
 
-    checkStackVersionState(stackId.getStackId(), CURRENT_VERSION, RepositoryVersionState.CURRENT);
+    assertRepoVersionState(stackId.getStackId(), CURRENT_VERSION, RepositoryVersionState.CURRENT);
 
     // Add ZK service with only ZOOKEEPER_SERVER
     List<String> hostList = new ArrayList<String>();
@@ -164,8 +154,9 @@ public class HostVersionOutOfSyncListenerTest {
         INSTALLED_VERSION);
     HostVersionEntity hv1 = helper.createHostVersion("h1", repositoryVersionEntity, RepositoryVersionState.INSTALLED);
     c1.recalculateAllClusterVersionStates();
-    checkStackVersionState(stackId.getStackId(), INSTALLED_VERSION, RepositoryVersionState.INSTALLED);
-    checkStackVersionState(stackId.getStackId(), CURRENT_VERSION, RepositoryVersionState.CURRENT);
+    assertRepoVersionState(stackId.getStackId(), INSTALLED_VERSION,
+        RepositoryVersionState.INSTALLED);
+    assertRepoVersionState(stackId.getStackId(), CURRENT_VERSION, RepositoryVersionState.CURRENT);
 
     // Add new host and verify that it has all host versions present
     List<HostVersionEntity> h2Versions = hostVersionDAO.findAll();
@@ -191,7 +182,8 @@ public class HostVersionOutOfSyncListenerTest {
             INSTALLED_VERSION);
     HostVersionEntity hv2 = helper.createHostVersion("h1", repositoryVersionEntity, RepositoryVersionState.INSTALLED);
     c1.recalculateAllClusterVersionStates();
-    checkStackVersionState(stackId.getStackId(), INSTALLED_VERSION, RepositoryVersionState.INSTALLED);
+    assertRepoVersionState(stackId.getStackId(), INSTALLED_VERSION,
+        RepositoryVersionState.INSTALLED);
 
     // Add new host and verify that it has all host versions present
     List<HostVersionEntity> h2Versions = hostVersionDAO.findAll();
@@ -214,7 +206,7 @@ public class HostVersionOutOfSyncListenerTest {
     String INSTALLED_VERSION = "2.2.0-1000";
     String INSTALLED_VERSION_2 = "2.1.1-2000";
     StackId stackId = new StackId(this.stackId);
-    StackId yaStackId = new StackId(this.yetAnotherStackId);
+    StackId yaStackId = new StackId(yetAnotherStackId);
 
     createClusterAndHosts(INSTALLED_VERSION, stackId);
     addRepoVersion(INSTALLED_VERSION_2, yaStackId);
@@ -239,8 +231,10 @@ public class HostVersionOutOfSyncListenerTest {
 
     List<HostVersionEntity> hostVersions = hostVersionDAO.findAll();
 
-    checkStackVersionState(stackId.getStackId(), INSTALLED_VERSION, RepositoryVersionState.INSTALLED);
-    checkStackVersionState(yaStackId.getStackId(), INSTALLED_VERSION_2, RepositoryVersionState.INSTALLED);
+    assertRepoVersionState(stackId.getStackId(), INSTALLED_VERSION,
+        RepositoryVersionState.INSTALLED);
+    assertRepoVersionState(yaStackId.getStackId(), INSTALLED_VERSION_2,
+        RepositoryVersionState.INSTALLED);
     for (HostVersionEntity hostVersionEntity : hostVersions) {
       if (hostVersionEntity.getRepositoryVersion().getVersion().equals(INSTALLED_VERSION) ||
               hostVersionEntity.getRepositoryVersion().getVersion().equals(INSTALLED_VERSION_2)) {
@@ -286,7 +280,8 @@ public class HostVersionOutOfSyncListenerTest {
     List<HostVersionEntity> hostVersions = hostVersionDAO.findAll();
 
     // Host version should not transition to OUT_OF_SYNC state
-    checkStackVersionState(stackId.getStackId(), INSTALLED_VERSION, RepositoryVersionState.INSTALLED);
+    assertRepoVersionState(stackId.getStackId(), INSTALLED_VERSION,
+        RepositoryVersionState.INSTALLED);
     for (HostVersionEntity hostVersionEntity : hostVersions) {
       if (hostVersionEntity.getRepositoryVersion().getVersion().equals(INSTALLED_VERSION)) {
         assertEquals(hostVersionEntity.getState(), RepositoryVersionState.INSTALLED);
@@ -304,7 +299,7 @@ public class HostVersionOutOfSyncListenerTest {
     String INSTALLED_VERSION = "2.2.0-1000";
     String INSTALLED_VERSION_2 = "2.1.1-2000";
     StackId stackId = new StackId(this.stackId);
-    StackId yaStackId = new StackId(this.yetAnotherStackId);
+    StackId yaStackId = new StackId(yetAnotherStackId);
 
     createClusterAndHosts(INSTALLED_VERSION, stackId);
     addRepoVersion(INSTALLED_VERSION_2, yaStackId);
@@ -322,7 +317,8 @@ public class HostVersionOutOfSyncListenerTest {
     changedHosts.add("h2");
     changedHosts.add("h3");
 
-    checkStackVersionState(stackId.getStackId(), INSTALLED_VERSION, RepositoryVersionState.INSTALLED);
+    assertRepoVersionState(stackId.getStackId(), INSTALLED_VERSION,
+        RepositoryVersionState.INSTALLED);
     List<HostVersionEntity> hostVersions = hostVersionDAO.findAll();
 
     for (HostVersionEntity hostVersionEntity : hostVersions) {
@@ -350,21 +346,21 @@ public class HostVersionOutOfSyncListenerTest {
     h1.setState(HostState.HEALTHY);
 
     StackId stackId = new StackId(this.stackId);
-    StackId yaStackId = new StackId(this.yetAnotherStackId);
+    StackId yaStackId = new StackId(yetAnotherStackId);
     RepositoryVersionEntity repositoryVersionEntity = helper.getOrCreateRepositoryVersion(stackId,"2.2.0-1000");
     RepositoryVersionEntity repositoryVersionEntity2 = helper.getOrCreateRepositoryVersion(stackId,"2.2.0-2000");
     c1.createClusterVersion(stackId, "2.2.0-1000", "admin", RepositoryVersionState.INSTALLING);
     c1.setCurrentStackVersion(stackId);
     c1.recalculateAllClusterVersionStates();
-    checkStackVersionState(stackId.getStackId(), "2.2.0-1000", RepositoryVersionState.INSTALLING);
-    checkStackVersionState(stackId.getStackId(), "2.2.0-2086", RepositoryVersionState.CURRENT);
+    assertRepoVersionState(stackId.getStackId(), "2.2.0-1000", RepositoryVersionState.INSTALLING);
+    assertRepoVersionState(stackId.getStackId(), "2.2.0-2086", RepositoryVersionState.CURRENT);
 
     HostVersionEntity hv1 = helper.createHostVersion("h1", repositoryVersionEntity, RepositoryVersionState.INSTALLED);
     HostVersionEntity hv2 = helper.createHostVersion("h1", repositoryVersionEntity2, RepositoryVersionState.INSTALLED);
     c1.recalculateAllClusterVersionStates();
-    checkStackVersionState(stackId.getStackId(), "2.2.0-1000", RepositoryVersionState.INSTALLED);
-    checkStackVersionState(stackId.getStackId(), "2.2.0-2000", RepositoryVersionState.INSTALLED);
-    checkStackVersionState(stackId.getStackId(), "2.2.0-2086", RepositoryVersionState.CURRENT);
+    assertRepoVersionState(stackId.getStackId(), "2.2.0-1000", RepositoryVersionState.INSTALLED);
+    assertRepoVersionState(stackId.getStackId(), "2.2.0-2000", RepositoryVersionState.INSTALLED);
+    assertRepoVersionState(stackId.getStackId(), "2.2.0-2086", RepositoryVersionState.CURRENT);
 
     // Add new host and verify that it has all host versions present
     addHost("h2");
@@ -381,7 +377,64 @@ public class HostVersionOutOfSyncListenerTest {
     }
   }
 
-  private void addHost(String hostname) throws AmbariException{
+  /**
+   * Tests that when a host is removed, the {@link HostRemovedEvent} fires and
+   * eventually calls to recalculate the cluster state.
+   */
+  @Test
+  public void testOnHostRemovedEvent() throws AmbariException {
+    // add the 2nd host
+    addHost("h2");
+    clusters.mapHostToCluster("h2", "c1");
+    clusters.getHost("h2").setState(HostState.HEALTHY);
+    clusters.getHost("h2").persist();
+
+    StackId stackId = new StackId(this.stackId);
+    RepositoryVersionEntity repositoryVersionEntity = helper.getOrCreateRepositoryVersion(stackId,
+        "2.2.9-9999");
+
+    c1.createClusterVersion(stackId, "2.2.9-9999", "admin", RepositoryVersionState.INSTALLING);
+    c1.setCurrentStackVersion(stackId);
+    c1.recalculateAllClusterVersionStates();
+
+    for (ClusterVersionEntity cve : c1.getAllClusterVersions()) {
+      System.out.println(cve.getRepositoryVersion().getDisplayName());
+    }
+
+    assertRepoVersionState(stackId.getStackId(), "2.2.0", RepositoryVersionState.CURRENT);
+    assertRepoVersionState(stackId.getStackId(), "2.2.9-9999", RepositoryVersionState.INSTALLING);
+
+    HostVersionEntity hv1 = helper.createHostVersion("h1", repositoryVersionEntity,
+        RepositoryVersionState.INSTALLED);
+    HostVersionEntity hv2 = helper.createHostVersion("h2", repositoryVersionEntity,
+        RepositoryVersionState.INSTALLED);
+
+    // do an initial calculate to make sure the new repo is installing
+    c1.recalculateAllClusterVersionStates();
+    assertRepoVersionState(stackId.getStackId(), "2.2.0", RepositoryVersionState.CURRENT);
+    assertRepoVersionState(stackId.getStackId(), "2.2.9-9999", RepositoryVersionState.INSTALLED);
+
+    // make it seems like we upgraded, but 1 host still hasn't finished
+    hv1.setState(RepositoryVersionState.UPGRADED);
+    hv2.setState(RepositoryVersionState.UPGRADING);
+    hostVersionDAO.merge(hv1);
+    hostVersionDAO.merge(hv2);
+
+    // recalculate and ensure that the cluster is UPGRADING
+    c1.recalculateAllClusterVersionStates();
+    assertRepoVersionState(stackId.getStackId(), "2.2.0", RepositoryVersionState.CURRENT);
+    assertRepoVersionState(stackId.getStackId(), "2.2.9-9999", RepositoryVersionState.UPGRADING);
+
+    // delete the host that was UPGRADING, and DON'T call recalculate; let the
+    // event handle it
+    injector.getInstance(UnitOfWork.class).begin();
+    clusters.deleteHost("h2");
+    injector.getInstance(UnitOfWork.class).end();
+    assertRepoVersionState(stackId.getStackId(), "2.2.0", RepositoryVersionState.CURRENT);
+    assertRepoVersionState(stackId.getStackId(), "2.2.9-9999", RepositoryVersionState.UPGRADED);
+  }
+
+  private void addHost(String hostname) throws AmbariException {
     clusters.addHost(hostname);
 
     Host host1 = clusters.getHost(hostname);
@@ -439,11 +492,13 @@ public class HostVersionOutOfSyncListenerTest {
     }
   }
 
-  private void checkStackVersionState(String stack, String version, RepositoryVersionState state) {
+  private void assertRepoVersionState(String stack, String version, RepositoryVersionState state) {
+    StackId stackId = new StackId(stack);
     Collection<ClusterVersionEntity> allClusterVersions = c1.getAllClusterVersions();
     for (ClusterVersionEntity entity : allClusterVersions) {
-      if (entity.getRepositoryVersion().getStack().equals(stack)
-              && entity.getRepositoryVersion().getVersion().equals(version)) {
+      StackId clusterVersionStackId = new StackId(entity.getRepositoryVersion().getStack());
+      if (clusterVersionStackId.equals(stackId)
+          && entity.getRepositoryVersion().getVersion().equals(version)) {
         assertEquals(state, entity.getState());
       }
     }

ambari-server/src/test/java/org/apache/ambari/server/orm/OrmTestHelper.java

@@ -35,7 +35,6 @@ import java.util.UUID;
 
 import javax.persistence.EntityManager;
 
-import junit.framework.Assert;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.Role;
 import org.apache.ambari.server.RoleCommand;
@@ -95,6 +94,8 @@ import com.google.inject.Provider;
 import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 
+import junit.framework.Assert;
+
 @Singleton
 public class OrmTestHelper {
 
@@ -626,11 +627,15 @@ public class OrmTestHelper {
    * Convenient method to create host version for given stack.
    */
   public HostVersionEntity createHostVersion(String hostName, RepositoryVersionEntity repositoryVersionEntity,
-                                             RepositoryVersionState repositoryVersionState) {
+      RepositoryVersionState repositoryVersionState) {
     HostEntity hostEntity = hostDAO.findByName(hostName);
     HostVersionEntity hostVersionEntity = new HostVersionEntity(hostEntity, repositoryVersionEntity, repositoryVersionState);
+    hostVersionEntity.setHostId(hostEntity.getHostId());
     hostVersionDAO.create(hostVersionEntity);
+
+    hostEntity.getHostVersionEntities().add(hostVersionEntity);
+    hostDAO.merge(hostEntity);
+
     return hostVersionEntity;
   }
-
 }