AMBARI-16410 - Ambari Should Suspend Alerts Notifications During Upgrade (jonathanhurley)

ambari-server/src/main/java/org/apache/ambari/server/events/listeners/alerts/AlertStateChangedListener.java

@@ -22,7 +22,9 @@ import java.util.List;
 import java.util.Set;
 import java.util.UUID;
 
+import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.EagerSingleton;
+import org.apache.ambari.server.controller.RootServiceResponseFactory.Services;
 import org.apache.ambari.server.events.AlertStateChangeEvent;
 import org.apache.ambari.server.events.publishers.AlertEventPublisher;
 import org.apache.ambari.server.orm.dao.AlertDispatchDAO;
@@ -35,14 +37,18 @@ import org.apache.ambari.server.orm.entities.AlertTargetEntity;
 import org.apache.ambari.server.state.Alert;
 import org.apache.ambari.server.state.AlertFirmness;
 import org.apache.ambari.server.state.AlertState;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.MaintenanceState;
 import org.apache.ambari.server.state.NotificationState;
+import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.eventbus.AllowConcurrentEvents;
 import com.google.common.eventbus.Subscribe;
 import com.google.inject.Inject;
+import com.google.inject.Provider;
 import com.google.inject.Singleton;
 
 /**
@@ -56,6 +62,13 @@ import com.google.inject.Singleton;
  * {@link AlertFirmness#HARD} for any notifications to be created. This is
  * because a SOFT non-OK alert (such as CRITICAL) would not have caused a
  * notification, so changing back from this SOFT state should not either.
+ * <p/>
+ * This class will not create {@link AlertNoticeEntity}s in the following cases:
+ * <ul>
+ * <li>If {@link AlertTargetEntity#isEnabled()} is {@code false}
+ * <li>If the cluster is upgrading or the upgrade is suspended, only
+ * {@link Services#AMBARI} alerts will be dispatched.
+ * </ul>
  */
 @Singleton
 @EagerSingleton
@@ -84,6 +97,12 @@ public class AlertStateChangedListener {
   @Inject
   private AlertDispatchDAO m_alertsDispatchDao;
 
+  /**
+   * Used to retrieve a cluster and check for upgrades in progress.
+   */
+  @Inject
+  private Provider<Clusters> m_clusters;
+
   /**
    * Constructor.
    *
@@ -151,7 +170,7 @@ public class AlertStateChangedListener {
       }
 
       for (AlertTargetEntity target : targets) {
-        if (!isAlertTargetInterestedAndEnabled(target, history)) {
+        if (!canDispatch(target, history, definition)) {
           continue;
         }
 
@@ -171,20 +190,26 @@ public class AlertStateChangedListener {
   }
 
   /**
-   * Gets whether the {@link AlertTargetEntity} is interested in receiving a
-   * notification about the {@link AlertHistoryEntity}'s state change. If an
-   * alert target is disabled, then this will return {@code false}.
+   * Gets whether an {@link AlertNoticeEntity} should be created for the
+   * {@link AlertHistoryEntity} and {@link AlertTargetEntity}. Reasons this
+   * would be false include:
+   * <ul>
+   * <li>The target is disabled
+   * <li>The target is not configured for the state of the alert
+   * <li>The cluster is upgrading and the alert is cluster-related
+   * </ul>
    *
    * @param target
    *          the target (not {@code null}).
    * @param history
    *          the history entry that represents the state change (not
    *          {@code null}).
-   * @return {@code true} if the target cares about this state change,
+   * @return {@code true} if a notification should be dispatched for the target,
    *         {@code false} otherwise.
+   * @see AlertTargetEntity#isEnabled()
    */
-  private boolean isAlertTargetInterestedAndEnabled(AlertTargetEntity target,
-      AlertHistoryEntity history) {
+  private boolean canDispatch(AlertTargetEntity target,
+      AlertHistoryEntity history, AlertDefinitionEntity definition) {
 
     // disable alert targets should be skipped
     if (!target.isEnabled()) {
@@ -198,6 +223,29 @@ public class AlertStateChangedListener {
       }
     }
 
+    // check if in an upgrade
+    Long clusterId = history.getClusterId();
+    try {
+      Cluster cluster = m_clusters.get().getClusterById(clusterId);
+      if (null != cluster.getUpgradeEntity() || cluster.isUpgradeSuspended()) {
+        // only send AMBARI alerts if in an upgrade
+        String serviceName = definition.getServiceName();
+        if (!StringUtils.equals(serviceName, Services.AMBARI.name())) {
+          LOG.debug(
+              "Skipping alert notifications for {} because the cluster is upgrading",
+              definition.getDefinitionName(), target);
+
+          return false;
+        }
+      }
+    } catch (AmbariException ambariException) {
+      LOG.warn(
+          "Unable to process an alert state change for cluster with ID {} because it does not exist",
+          clusterId);
+
+      return false;
+    }
+
     return true;
   }
 }

ambari-server/src/test/java/org/apache/ambari/server/state/alerts/AlertStateChangedEventTest.java

@@ -23,6 +23,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.ambari.server.controller.RootServiceResponseFactory.Services;
 import org.apache.ambari.server.events.AggregateAlertRecalculateEvent;
 import org.apache.ambari.server.events.AlertEvent;
 import org.apache.ambari.server.events.AlertStateChangeEvent;
@@ -38,12 +39,16 @@ import org.apache.ambari.server.orm.entities.AlertGroupEntity;
 import org.apache.ambari.server.orm.entities.AlertHistoryEntity;
 import org.apache.ambari.server.orm.entities.AlertNoticeEntity;
 import org.apache.ambari.server.orm.entities.AlertTargetEntity;
+import org.apache.ambari.server.orm.entities.UpgradeEntity;
 import org.apache.ambari.server.state.Alert;
 import org.apache.ambari.server.state.AlertFirmness;
 import org.apache.ambari.server.state.AlertState;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.MaintenanceState;
 import org.apache.ambari.server.utils.EventBusSynchronizer;
 import org.easymock.EasyMock;
+import org.easymock.EasyMockSupport;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -63,7 +68,7 @@ import junit.framework.Assert;
  * should only be created when received alerts which have a firmness of
  * {@link AlertFirmness#HARD}.
  */
-public class AlertStateChangedEventTest {
+public class AlertStateChangedEventTest extends EasyMockSupport {
 
   private AlertEventPublisher eventPublisher;
   private AlertDispatchDAO dispatchDao;
@@ -80,7 +85,6 @@ public class AlertStateChangedEventTest {
 
     injector.getInstance(GuiceJpaInitializer.class);
     m_listener = injector.getInstance(MockEventListener.class);
-
     dispatchDao = injector.getInstance(AlertDispatchDAO.class);
 
     // !!! need a synchronous op for testing
@@ -106,9 +110,13 @@ public class AlertStateChangedEventTest {
    * @throws Exception
    */
   @Test
+  @SuppressWarnings("unchecked")
   public void testAlertNoticeCreationFromEvent() throws Exception {
-    AlertTargetEntity alertTarget = EasyMock.createNiceMock(AlertTargetEntity.class);
-    AlertGroupEntity alertGroup = EasyMock.createMock(AlertGroupEntity.class);
+    // expect the normal calls which get a cluster by its ID
+    expectNormalCluster();
+
+    AlertTargetEntity alertTarget = createNiceMock(AlertTargetEntity.class);
+    AlertGroupEntity alertGroup = createMock(AlertGroupEntity.class);
     List<AlertGroupEntity> groups = new ArrayList<AlertGroupEntity>();
     Set<AlertTargetEntity> targets = new HashSet<AlertTargetEntity>();
 
@@ -124,20 +132,19 @@ public class AlertStateChangedEventTest {
         dispatchDao.findGroupsByDefinition(EasyMock.anyObject(AlertDefinitionEntity.class))).andReturn(
         groups).once();
 
-    dispatchDao.createNotices(EasyMock.<List<AlertNoticeEntity>>anyObject());
+    dispatchDao.createNotices((List<AlertNoticeEntity>) EasyMock.anyObject());
     EasyMock.expectLastCall().once();
 
-    EasyMock.replay(alertTarget, alertGroup, dispatchDao);
-
     AlertDefinitionEntity definition = getMockAlertDefinition();
 
     AlertCurrentEntity current = getMockedAlertCurrentEntity();
-    AlertHistoryEntity history = EasyMock.createNiceMock(AlertHistoryEntity.class);
-    AlertStateChangeEvent event = EasyMock.createNiceMock(AlertStateChangeEvent.class);
-    Alert alert = EasyMock.createNiceMock(Alert.class);
+    AlertHistoryEntity history = createNiceMock(AlertHistoryEntity.class);
+    AlertStateChangeEvent event = createNiceMock(AlertStateChangeEvent.class);
+    Alert alert = createNiceMock(Alert.class);
 
     EasyMock.expect(current.getAlertHistory()).andReturn(history).anyTimes();
     EasyMock.expect(current.getFirmness()).andReturn(AlertFirmness.HARD).atLeastOnce();
+    EasyMock.expect(history.getClusterId()).andReturn(1L).atLeastOnce();
     EasyMock.expect(history.getAlertState()).andReturn(AlertState.CRITICAL).atLeastOnce();
     EasyMock.expect(history.getAlertDefinition()).andReturn(definition).atLeastOnce();
     EasyMock.expect(alert.getText()).andReturn("The HDFS Foo Alert Is Not Good").atLeastOnce();
@@ -146,11 +153,9 @@ public class AlertStateChangedEventTest {
     EasyMock.expect(event.getNewHistoricalEntry()).andReturn(history).atLeastOnce();
     EasyMock.expect(event.getAlert()).andReturn(alert).atLeastOnce();
 
-    EasyMock.replay(definition, current, history, event, alert);
-
-    // async publishing
+    replayAll();
     eventPublisher.publish(event);
-    EasyMock.verify(dispatchDao, alertTarget, current, history, event);
+    verifyAll();
   }
 
   /**
@@ -161,8 +166,8 @@ public class AlertStateChangedEventTest {
    */
   @Test
   public void testAlertNoticeSkippedForTarget() throws Exception {
-    AlertTargetEntity alertTarget = EasyMock.createMock(AlertTargetEntity.class);
-    AlertGroupEntity alertGroup = EasyMock.createMock(AlertGroupEntity.class);
+    AlertTargetEntity alertTarget = createMock(AlertTargetEntity.class);
+    AlertGroupEntity alertGroup = createMock(AlertGroupEntity.class);
     List<AlertGroupEntity> groups = new ArrayList<AlertGroupEntity>();
     Set<AlertTargetEntity> targets = new HashSet<AlertTargetEntity>();
 
@@ -175,19 +180,17 @@ public class AlertStateChangedEventTest {
         EnumSet.of(AlertState.OK, AlertState.CRITICAL)).atLeastOnce();
 
     EasyMock.expect(
-        dispatchDao.findGroupsByDefinition(EasyMock.anyObject(AlertDefinitionEntity.class))).andReturn(
+        dispatchDao.findGroupsByDefinition(
+            EasyMock.anyObject(AlertDefinitionEntity.class))).andReturn(
         groups).once();
 
-    // dispatchDao should be strict enough to throw an exception on verify
-    // that the create alert notice method was not called
-    EasyMock.replay(alertTarget, alertGroup, dispatchDao);
 
     AlertDefinitionEntity definition = getMockAlertDefinition();
 
     AlertCurrentEntity current = getMockedAlertCurrentEntity();
-    AlertHistoryEntity history = EasyMock.createNiceMock(AlertHistoryEntity.class);
-    AlertStateChangeEvent event = EasyMock.createNiceMock(AlertStateChangeEvent.class);
-    Alert alert = EasyMock.createNiceMock(Alert.class);
+    AlertHistoryEntity history = createNiceMock(AlertHistoryEntity.class);
+    AlertStateChangeEvent event = createNiceMock(AlertStateChangeEvent.class);
+    Alert alert = createNiceMock(Alert.class);
 
     EasyMock.expect(current.getAlertHistory()).andReturn(history).anyTimes();
     EasyMock.expect(current.getFirmness()).andReturn(AlertFirmness.HARD).atLeastOnce();
@@ -195,7 +198,6 @@ public class AlertStateChangedEventTest {
     // use WARNING to ensure that the target (which only cares about OK/CRIT)
     // does not receive the alert notice
     EasyMock.expect(history.getAlertState()).andReturn(AlertState.WARNING).atLeastOnce();
-
     EasyMock.expect(history.getAlertDefinition()).andReturn(definition).atLeastOnce();
     EasyMock.expect(alert.getText()).andReturn("The HDFS Foo Alert Is Not Good").atLeastOnce();
     EasyMock.expect(alert.getState()).andReturn(AlertState.WARNING).atLeastOnce();
@@ -203,11 +205,9 @@ public class AlertStateChangedEventTest {
     EasyMock.expect(event.getNewHistoricalEntry()).andReturn(history).atLeastOnce();
     EasyMock.expect(event.getAlert()).andReturn(alert).atLeastOnce();
 
-    EasyMock.replay(definition, current, history, event, alert);
-
-    // async publishing
+    replayAll();
     eventPublisher.publish(event);
-    EasyMock.verify(dispatchDao, current, history, event);
+    verifyAll();
   }
 
   /**
@@ -218,8 +218,8 @@ public class AlertStateChangedEventTest {
    */
   @Test
   public void testAlertNoticeSkippedForDisabledTarget() throws Exception {
-    AlertTargetEntity alertTarget = EasyMock.createMock(AlertTargetEntity.class);
-    AlertGroupEntity alertGroup = EasyMock.createMock(AlertGroupEntity.class);
+    AlertTargetEntity alertTarget = createMock(AlertTargetEntity.class);
+    AlertGroupEntity alertGroup = createMock(AlertGroupEntity.class);
     List<AlertGroupEntity> groups = new ArrayList<AlertGroupEntity>();
     Set<AlertTargetEntity> targets = new HashSet<AlertTargetEntity>();
 
@@ -229,19 +229,18 @@ public class AlertStateChangedEventTest {
     EasyMock.expect(alertGroup.getAlertTargets()).andReturn(targets).once();
     EasyMock.expect(alertTarget.isEnabled()).andReturn(Boolean.FALSE).atLeastOnce();
 
-    EasyMock.expect(dispatchDao.findGroupsByDefinition(
-        EasyMock.anyObject(AlertDefinitionEntity.class))).andReturn(groups).once();
+    EasyMock.expect(
+        dispatchDao.findGroupsByDefinition(
+            EasyMock.anyObject(AlertDefinitionEntity.class))).andReturn(
+        groups).once();
 
-    // dispatchDao should be strict enough to throw an exception on verify
-    // that the create alert notice method was not called
-    EasyMock.replay(alertTarget, alertGroup, dispatchDao);
 
     AlertDefinitionEntity definition = getMockAlertDefinition();
 
     AlertCurrentEntity current = getMockedAlertCurrentEntity();
-    AlertHistoryEntity history = EasyMock.createNiceMock(AlertHistoryEntity.class);
-    AlertStateChangeEvent event = EasyMock.createNiceMock(AlertStateChangeEvent.class);
-    Alert alert = EasyMock.createNiceMock(Alert.class);
+    AlertHistoryEntity history = createNiceMock(AlertHistoryEntity.class);
+    AlertStateChangeEvent event = createNiceMock(AlertStateChangeEvent.class);
+    Alert alert = createNiceMock(Alert.class);
 
     EasyMock.expect(current.getAlertHistory()).andReturn(history).anyTimes();
     EasyMock.expect(current.getFirmness()).andReturn(AlertFirmness.HARD).atLeastOnce();
@@ -254,11 +253,9 @@ public class AlertStateChangedEventTest {
     EasyMock.expect(event.getNewHistoricalEntry()).andReturn(history).atLeastOnce();
     EasyMock.expect(event.getAlert()).andReturn(alert).atLeastOnce();
 
-    EasyMock.replay(definition, current, history, event, alert);
-
-    // async publishing
+    replayAll();
     eventPublisher.publish(event);
-    EasyMock.verify(dispatchDao, alertTarget, current, history, event);
+    verifyAll();
   }
 
   /**
@@ -269,14 +266,12 @@ public class AlertStateChangedEventTest {
    */
   @Test
   public void testSoftAlertDoesNotCreateNotifications() throws Exception {
-    EasyMock.replay(dispatchDao);
-
     AlertDefinitionEntity definition = getMockAlertDefinition();
 
     AlertCurrentEntity current = getMockedAlertCurrentEntity();
-    AlertHistoryEntity history = EasyMock.createNiceMock(AlertHistoryEntity.class);
-    AlertStateChangeEvent event = EasyMock.createNiceMock(AlertStateChangeEvent.class);
-    Alert alert = EasyMock.createNiceMock(Alert.class);
+    AlertHistoryEntity history = createNiceMock(AlertHistoryEntity.class);
+    AlertStateChangeEvent event = createNiceMock(AlertStateChangeEvent.class);
+    Alert alert = createNiceMock(Alert.class);
 
     // make the alert SOFT so that no notifications are sent
     EasyMock.expect(current.getAlertHistory()).andReturn(history).anyTimes();
@@ -289,11 +284,9 @@ public class AlertStateChangedEventTest {
     EasyMock.expect(event.getNewHistoricalEntry()).andReturn(history).atLeastOnce();
     EasyMock.expect(event.getAlert()).andReturn(alert).atLeastOnce();
 
-    EasyMock.replay(definition, current, history, event, alert);
-
-    // async publishing
+    replayAll();
     eventPublisher.publish(event);
-    EasyMock.verify(dispatchDao, current, history, event);
+    verifyAll();
   }
 
   /**
@@ -305,14 +298,12 @@ public class AlertStateChangedEventTest {
    */
   @Test
   public void testSoftAlertTransitionToHardOKDoesNotCreateNotification() throws Exception {
-    EasyMock.replay(dispatchDao);
-
     AlertDefinitionEntity definition = getMockAlertDefinition();
 
     AlertCurrentEntity current = getMockedAlertCurrentEntity();
-    AlertHistoryEntity history = EasyMock.createNiceMock(AlertHistoryEntity.class);
-    AlertStateChangeEvent event = EasyMock.createNiceMock(AlertStateChangeEvent.class);
-    Alert alert = EasyMock.createNiceMock(Alert.class);
+    AlertHistoryEntity history = createNiceMock(AlertHistoryEntity.class);
+    AlertStateChangeEvent event = createNiceMock(AlertStateChangeEvent.class);
+    Alert alert = createNiceMock(Alert.class);
 
     // register a HARD/OK for the brand new alert coming in
     EasyMock.expect(current.getAlertHistory()).andReturn(history).anyTimes();
@@ -330,11 +321,9 @@ public class AlertStateChangedEventTest {
     EasyMock.expect(event.getNewHistoricalEntry()).andReturn(history).atLeastOnce();
     EasyMock.expect(event.getAlert()).andReturn(alert).atLeastOnce();
 
-    EasyMock.replay(definition, current, history, event, alert);
-
-    // async publishing
+    replayAll();
     eventPublisher.publish(event);
-    EasyMock.verify(dispatchDao, current, history, event);
+    verifyAll();
   }
 
   /**
@@ -343,11 +332,11 @@ public class AlertStateChangedEventTest {
    * @return
    */
   private AlertDefinitionEntity getMockAlertDefinition() {
-    AlertDefinitionEntity definition = EasyMock.createNiceMock(AlertDefinitionEntity.class);
-    EasyMock.expect(definition.getDefinitionId()).andReturn(1L);
-    EasyMock.expect(definition.getServiceName()).andReturn("HDFS");
-    EasyMock.expect(definition.getLabel()).andReturn("hdfs-foo-alert");
-    EasyMock.expect(definition.getDescription()).andReturn("HDFS Foo Alert");
+    AlertDefinitionEntity definition = createNiceMock(AlertDefinitionEntity.class);
+    EasyMock.expect(definition.getDefinitionId()).andReturn(1L).anyTimes();
+    EasyMock.expect(definition.getServiceName()).andReturn("HDFS").anyTimes();
+    EasyMock.expect(definition.getLabel()).andReturn("hdfs-foo-alert").anyTimes();
+    EasyMock.expect(definition.getDescription()).andReturn("HDFS Foo Alert").anyTimes();
 
     return definition;
   }
@@ -358,7 +347,7 @@ public class AlertStateChangedEventTest {
    * @return
    */
   private AlertCurrentEntity getMockedAlertCurrentEntity() {
-    AlertCurrentEntity current = EasyMock.createNiceMock(AlertCurrentEntity.class);
+    AlertCurrentEntity current = createNiceMock(AlertCurrentEntity.class);
     EasyMock.expect(current.getMaintenanceState()).andReturn(MaintenanceState.OFF).anyTimes();
     return current;
   }
@@ -379,6 +368,209 @@ public class AlertStateChangedEventTest {
     Assert.assertEquals(1, m_listener.getAlertEventReceivedCount(eventClass));
   }
 
+  /**
+   * Tests that no {@link AlertNoticeEntity} instances are created during an
+   * upgrade.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testUpgradingClusterSkipsAlerts() throws Exception {
+    // expect an upgrading cluster
+    expectUpgradingCluster();
+
+    AlertTargetEntity alertTarget = createNiceMock(AlertTargetEntity.class);
+    AlertGroupEntity alertGroup = createMock(AlertGroupEntity.class);
+    List<AlertGroupEntity> groups = new ArrayList<AlertGroupEntity>();
+    Set<AlertTargetEntity> targets = new HashSet<AlertTargetEntity>();
+
+    targets.add(alertTarget);
+    groups.add(alertGroup);
+
+    EasyMock.expect(alertGroup.getAlertTargets()).andReturn(targets).once();
+    EasyMock.expect(alertTarget.isEnabled()).andReturn(Boolean.TRUE).atLeastOnce();
+    EasyMock.expect(alertTarget.getAlertStates()).andReturn(EnumSet.allOf(AlertState.class)).atLeastOnce();
+
+    EasyMock.expect(dispatchDao.findGroupsByDefinition(
+        EasyMock.anyObject(AlertDefinitionEntity.class))).andReturn(groups).once();
+
+    AlertDefinitionEntity definition = getMockAlertDefinition();
+
+    AlertCurrentEntity current = getMockedAlertCurrentEntity();
+    AlertHistoryEntity history = createNiceMock(AlertHistoryEntity.class);
+    AlertStateChangeEvent event = createNiceMock(AlertStateChangeEvent.class);
+    Alert alert = createNiceMock(Alert.class);
+
+    EasyMock.expect(current.getAlertHistory()).andReturn(history).anyTimes();
+    EasyMock.expect(current.getFirmness()).andReturn(AlertFirmness.HARD).atLeastOnce();
+    EasyMock.expect(history.getClusterId()).andReturn(1L).atLeastOnce();
+    EasyMock.expect(history.getAlertState()).andReturn(AlertState.CRITICAL).atLeastOnce();
+    EasyMock.expect(history.getAlertDefinition()).andReturn(definition).atLeastOnce();
+    EasyMock.expect(alert.getText()).andReturn("The HDFS Foo Alert Is Not Good").atLeastOnce();
+    EasyMock.expect(alert.getState()).andReturn(AlertState.CRITICAL).atLeastOnce();
+    EasyMock.expect(event.getCurrentAlert()).andReturn(current).atLeastOnce();
+    EasyMock.expect(event.getNewHistoricalEntry()).andReturn(history).atLeastOnce();
+    EasyMock.expect(event.getAlert()).andReturn(alert).atLeastOnce();
+
+    replayAll();
+    eventPublisher.publish(event);
+    verifyAll();
+  }
+
+  /**
+   * Tests that no {@link AlertNoticeEntity} instances are created when a
+   * cluster upgrade is suspended.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testUpgradeSuspendedClusterSkipsAlerts() throws Exception {
+    // expect an upgrading cluster
+    expectUpgradeSuspendedCluster();
+
+    AlertTargetEntity alertTarget = createNiceMock(AlertTargetEntity.class);
+    AlertGroupEntity alertGroup = createMock(AlertGroupEntity.class);
+    List<AlertGroupEntity> groups = new ArrayList<AlertGroupEntity>();
+    Set<AlertTargetEntity> targets = new HashSet<AlertTargetEntity>();
+
+    targets.add(alertTarget);
+    groups.add(alertGroup);
+
+    EasyMock.expect(alertGroup.getAlertTargets()).andReturn(targets).once();
+    EasyMock.expect(alertTarget.isEnabled()).andReturn(Boolean.TRUE).atLeastOnce();
+    EasyMock.expect(alertTarget.getAlertStates()).andReturn(
+        EnumSet.allOf(AlertState.class)).atLeastOnce();
+
+    EasyMock.expect(dispatchDao.findGroupsByDefinition(
+        EasyMock.anyObject(AlertDefinitionEntity.class))).andReturn(groups).once();
+
+    AlertDefinitionEntity definition = getMockAlertDefinition();
+
+    AlertCurrentEntity current = getMockedAlertCurrentEntity();
+    AlertHistoryEntity history = createNiceMock(AlertHistoryEntity.class);
+    AlertStateChangeEvent event = createNiceMock(AlertStateChangeEvent.class);
+    Alert alert = createNiceMock(Alert.class);
+
+    EasyMock.expect(current.getAlertHistory()).andReturn(history).anyTimes();
+    EasyMock.expect(current.getFirmness()).andReturn(AlertFirmness.HARD).atLeastOnce();
+    EasyMock.expect(history.getClusterId()).andReturn(1L).atLeastOnce();
+    EasyMock.expect(history.getAlertState()).andReturn(AlertState.CRITICAL).atLeastOnce();
+    EasyMock.expect(history.getAlertDefinition()).andReturn(definition).atLeastOnce();
+    EasyMock.expect(alert.getText()).andReturn("The HDFS Foo Alert Is Not Good").atLeastOnce();
+    EasyMock.expect(alert.getState()).andReturn(AlertState.CRITICAL).atLeastOnce();
+    EasyMock.expect(event.getCurrentAlert()).andReturn(current).atLeastOnce();
+    EasyMock.expect(event.getNewHistoricalEntry()).andReturn(history).atLeastOnce();
+    EasyMock.expect(event.getAlert()).andReturn(alert).atLeastOnce();
+
+    replayAll();
+    eventPublisher.publish(event);
+    verifyAll();
+  }
+
+  /**
+   * Tests that {@link AlertNoticeEntity} instances are created during an
+   * upgrade as long as they are for the AMBARI service.
+   *
+   * @throws Exception
+   */
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testAmbariAlertsSendDuringUpgrade() throws Exception {
+    // expect an upgrading cluster
+    expectUpgradingCluster();
+
+    AlertTargetEntity alertTarget = createNiceMock(AlertTargetEntity.class);
+    AlertGroupEntity alertGroup = createMock(AlertGroupEntity.class);
+    List<AlertGroupEntity> groups = new ArrayList<AlertGroupEntity>();
+    Set<AlertTargetEntity> targets = new HashSet<AlertTargetEntity>();
+
+    targets.add(alertTarget);
+    groups.add(alertGroup);
+
+    EasyMock.expect(alertGroup.getAlertTargets()).andReturn(targets).once();
+    EasyMock.expect(alertTarget.isEnabled()).andReturn(Boolean.TRUE).atLeastOnce();
+    EasyMock.expect(alertTarget.getAlertStates()).andReturn(
+        EnumSet.allOf(AlertState.class)).atLeastOnce();
+
+    EasyMock.expect(dispatchDao.findGroupsByDefinition(
+        EasyMock.anyObject(AlertDefinitionEntity.class))).andReturn(groups).once();
+
+    // create the definition for the AMBARI service
+    AlertDefinitionEntity definition = createNiceMock(AlertDefinitionEntity.class);
+    EasyMock.expect(definition.getDefinitionId()).andReturn(1L).anyTimes();
+    EasyMock.expect(definition.getServiceName()).andReturn(Services.AMBARI.name()).anyTimes();
+    EasyMock.expect(definition.getLabel()).andReturn("ambari-foo-alert").anyTimes();
+    EasyMock.expect(definition.getDescription()).andReturn("Ambari Foo Alert").anyTimes();
+
+    dispatchDao.createNotices((List<AlertNoticeEntity>) EasyMock.anyObject());
+
+    AlertCurrentEntity current = getMockedAlertCurrentEntity();
+    AlertHistoryEntity history = createNiceMock(AlertHistoryEntity.class);
+    AlertStateChangeEvent event = createNiceMock(AlertStateChangeEvent.class);
+    Alert alert = createNiceMock(Alert.class);
+
+    EasyMock.expect(current.getAlertHistory()).andReturn(history).anyTimes();
+    EasyMock.expect(current.getFirmness()).andReturn(AlertFirmness.HARD).atLeastOnce();
+    EasyMock.expect(history.getClusterId()).andReturn(1L).atLeastOnce();
+    EasyMock.expect(history.getAlertState()).andReturn(AlertState.CRITICAL).atLeastOnce();
+    EasyMock.expect(history.getAlertDefinition()).andReturn(definition).atLeastOnce();
+    EasyMock.expect(alert.getText()).andReturn("The HDFS Foo Alert Is Not Good").atLeastOnce();
+    EasyMock.expect(alert.getState()).andReturn(AlertState.CRITICAL).atLeastOnce();
+    EasyMock.expect(event.getCurrentAlert()).andReturn(current).atLeastOnce();
+    EasyMock.expect(event.getNewHistoricalEntry()).andReturn(history).atLeastOnce();
+    EasyMock.expect(event.getAlert()).andReturn(alert).atLeastOnce();
+
+    replayAll();
+    eventPublisher.publish(event);
+    verifyAll();
+  }
+
+  /**
+   * Create expectations around a normal cluster.
+   *
+   * @throws Exception
+   */
+  private void expectNormalCluster() throws Exception {
+    Clusters clusters = injector.getInstance(Clusters.class);
+    Cluster cluster = createMock(Cluster.class);
+
+    EasyMock.expect(clusters.getClusterById(EasyMock.anyLong())).andReturn(cluster).atLeastOnce();
+    EasyMock.expect(cluster.getUpgradeEntity()).andReturn(null).anyTimes();
+    EasyMock.expect(cluster.isUpgradeSuspended()).andReturn(false).anyTimes();
+  }
+
+  /**
+   * Create expectations around a cluster in an active upgrade.
+   *
+   * @throws Exception
+   */
+  private void expectUpgradingCluster() throws Exception {
+    Clusters clusters = injector.getInstance(Clusters.class);
+    Cluster cluster = createMock(Cluster.class);
+
+    EasyMock.reset(clusters);
+
+    EasyMock.expect(clusters.getClusterById(EasyMock.anyLong())).andReturn(cluster).atLeastOnce();
+    EasyMock.expect(cluster.getUpgradeEntity()).andReturn(new UpgradeEntity()).anyTimes();
+    EasyMock.expect(cluster.isUpgradeSuspended()).andReturn(false).anyTimes();
+  }
+
+  /**
+   * Create expectations around a cluster where the upgrade was suspended.
+   *
+   * @throws Exception
+   */
+  private void expectUpgradeSuspendedCluster() throws Exception {
+    Cluster cluster = createMock(Cluster.class);
+    Clusters clusters = injector.getInstance(Clusters.class);
+
+    EasyMock.reset(clusters);
+
+    EasyMock.expect(clusters.getClusterById(EasyMock.anyLong())).andReturn(cluster).atLeastOnce();
+    EasyMock.expect(cluster.getUpgradeEntity()).andReturn(null).anyTimes();
+    EasyMock.expect(cluster.isUpgradeSuspended()).andReturn(true).anyTimes();
+  }
+
   /**
    *
    */
@@ -388,8 +580,12 @@ public class AlertStateChangedEventTest {
      */
     @Override
     public void configure(Binder binder) {
-      AlertDispatchDAO dispatchDao = EasyMock.createMock(AlertDispatchDAO.class);
-      binder.bind(AlertDispatchDAO.class).toInstance(dispatchDao);
+      Clusters clusters = createMock(Clusters.class);
+
+      // dispatchDao should be strict enough to throw an exception on verify
+      // that the create alert notice method was not called
+      binder.bind(AlertDispatchDAO.class).toInstance(createMock(AlertDispatchDAO.class));
+      binder.bind(Clusters.class).toInstance(clusters);
     }
   }
 }

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_alert_metrics_deviation.py

@@ -27,10 +27,6 @@ from mock.mock import patch, MagicMock
 # Local imports
 from stacks.utils.RMFTestCase import *
 
-# set locale for formatted strings from this alert
-import locale
-locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
-
 COMMON_SERVICES_ALERTS_DIR = "HDFS/2.1.0.2.0/package/alerts"
 
 file_path = os.path.dirname(os.path.abspath(__file__))
@@ -126,14 +122,14 @@ class TestAlertMetricsDeviation(RMFTestCase):
     [status, messages] = alert.execute(configurations=configs, parameters=parameters)
     self.assertEqual(status, RESULT_STATE_OK)
     self.assertTrue(messages is not None and len(messages) == 1)
-    self.assertEquals('The variance for this alert is 7,071ms which is within 100% of the 45,000ms average (45,000ms is the limit)', messages[0])
+    self.assertEquals('The variance for this alert is 7071ms which is within 100% of the 45000ms average (45000ms is the limit)', messages[0])
 
     # Warning
     response.read.return_value = '{"metrics":[{"metricname":"metric1","metrics":{"1459966360838":40000,"1459966370838":1000000}}]}'
     [status, messages] = alert.execute(configurations=configs, parameters=parameters)
     self.assertEqual(status, RESULT_STATE_WARNING)
     self.assertTrue(messages is not None and len(messages) == 1)
-    self.assertEquals('The variance for this alert is 678,823ms which is 131% of the 520,000ms average (520,000ms is the limit)', messages[0])
+    self.assertEquals('The variance for this alert is 678823ms which is 131% of the 520000ms average (520000ms is the limit)', messages[0])
 
     # HTTP request to AMS failed
     response.read.return_value = ''