AMBARI-15745 - Create Alert For Reporting Potential Issues With Slow REST Responses (jonathanhurley)

ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionManager.java

@@ -223,7 +223,9 @@ public class ActionManager {
     for (Request logicalRequest : topologyManager.getRequests(Collections.<Long>emptySet())) {
       //todo: Request.getStatus() returns HostRoleStatus and we are comparing to RequestStatus
       //todo: for now just compare the names as RequestStatus names are a subset of HostRoleStatus names
-      if (status == null || logicalRequest.getStatus().name().equals(status.name())) {
+      HostRoleStatus logicalRequestStatus = logicalRequest.getStatus();
+      if (status == null || (logicalRequestStatus != null
+          && logicalRequest.getStatus().name().equals(status.name()))) {
         requests.add(logicalRequest.getRequestId());
       }
     }

ambari-server/src/main/java/org/apache/ambari/server/alerts/AgentHeartbeatAlertRunnable.java

@@ -18,45 +18,25 @@
 package org.apache.ambari.server.alerts;
 
 import java.text.MessageFormat;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.ambari.server.events.AlertEvent;
-import org.apache.ambari.server.events.AlertReceivedEvent;
-import org.apache.ambari.server.events.publishers.AlertEventPublisher;
-import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
 import org.apache.ambari.server.state.Alert;
 import org.apache.ambari.server.state.AlertState;
 import org.apache.ambari.server.state.Cluster;
-import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Host;
 import org.apache.ambari.server.state.HostState;
 import org.apache.ambari.server.state.services.AmbariServerAlertService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.inject.Inject;
-import com.google.inject.Provider;
 
 /**
  * The {@link AgentHeartbeatAlertRunnable} is used by the
  * {@link AmbariServerAlertService} to check agent heartbeats and fire alert
  * events when agents are not reachable.
  */
-public class AgentHeartbeatAlertRunnable implements Runnable {
-
-  /**
-   * Logger.
-   */
-  private final static Logger LOG = LoggerFactory.getLogger(AgentHeartbeatAlertRunnable.class);
-
-  /**
-   * The unique name for the alert definition that governs this service.
-   */
-  private static final String HEARTBEAT_DEFINITION_NAME = "ambari_server_agent_heartbeat";
-
+public class AgentHeartbeatAlertRunnable extends AlertRunnable {
   /**
    * Agent initializing message.
    */
@@ -88,99 +68,66 @@ public class AgentHeartbeatAlertRunnable implements Runnable {
   private static final String UNKNOWN_MSG = "{0} has an unknown state of {1}";
 
   /**
-   * Used for looking up alert definitions.
+   * Constructor.
+   *
+   * @param definitionName
    */
-  @Inject
-  private AlertDefinitionDAO m_dao;
+  public AgentHeartbeatAlertRunnable(String definitionName) {
+    super(definitionName);
+  }
 
   /**
-   * Used to get alert definitions to use when generating alert instances.
+   * {@inheritDoc}
    */
-  @Inject
-  private Provider<Clusters> m_clustersProvider;
+  @Override
+  List<Alert> execute(Cluster cluster, AlertDefinitionEntity definition)
+      throws AmbariException {
+    long alertTimestamp = System.currentTimeMillis();
+
+    Collection<Host> hosts = cluster.getHosts();
+
+    List<Alert> alerts = new ArrayList<>(hosts.size());
+    for (Host host : hosts) {
+      String hostName = host.getHostName();
+
+      String alertText;
+      AlertState alertState = AlertState.OK;
+      HostState hostState = host.getState();
+
+      switch (hostState) {
+        case INIT:
+          alertText = MessageFormat.format(INIT_MSG, hostName);
+          break;
+        case HEALTHY:
+          alertText = MessageFormat.format(HEALTHY_MSG, hostName);
+          break;
+        case WAITING_FOR_HOST_STATUS_UPDATES:
+          alertText = MessageFormat.format(STATUS_UPDATE_MSG, hostName);
+          break;
+        case HEARTBEAT_LOST:
+          alertState = AlertState.CRITICAL;
+          alertText = MessageFormat.format(HEARTBEAT_LOST_MSG, hostName);
+          break;
+        case UNHEALTHY:
+          alertState = AlertState.CRITICAL;
+          alertText = MessageFormat.format(UNHEALTHY_MSG, hostName);
+        default:
+          alertState = AlertState.UNKNOWN;
+          alertText = MessageFormat.format(UNKNOWN_MSG, hostName, hostState);
+          break;
+      }
 
-  /**
-   * Publishes {@link AlertEvent} instances.
-   */
-  @Inject
-  private AlertEventPublisher m_alertEventPublisher;
+      Alert alert = new Alert(definition.getDefinitionName(), null, definition.getServiceName(),
+          definition.getComponentName(), hostName, alertState);
 
-  /**
-   * Constructor. Required for type introspection by
-   * {@link AmbariServerAlertService}.
-   */
-  public AgentHeartbeatAlertRunnable() {
-  }
+      alert.setLabel(definition.getLabel());
+      alert.setText(alertText);
+      alert.setTimestamp(alertTimestamp);
+      alert.setCluster(cluster.getClusterName());
 
-  @Override
-  public void run() {
-    try {
-      Map<String, Cluster> clusterMap = m_clustersProvider.get().getClusters();
-      for (Cluster cluster : clusterMap.values()) {
-        AlertDefinitionEntity entity = m_dao.findByName(cluster.getClusterId(),
-            HEARTBEAT_DEFINITION_NAME);
-
-        // skip this cluster if the runnable's alert definition is missing or
-        // disabled
-        if (null == entity || !entity.getEnabled()) {
-          continue;
-        }
-
-        long alertTimestamp = System.currentTimeMillis();
-
-        Map<String, Host> hostMap = m_clustersProvider.get().getHostsForCluster(
-            cluster.getClusterName());
-
-        Set<Entry<String, Host>> entries = hostMap.entrySet();
-        for (Entry<String, Host> entry : entries) {
-          String hostName = entry.getKey();
-          Host host = entry.getValue();
-
-          String alertText;
-          AlertState alertState = AlertState.OK;
-          HostState hostState = host.getState();
-
-          switch (hostState) {
-            case INIT:
-              alertText = MessageFormat.format(INIT_MSG, hostName);
-              break;
-            case HEALTHY:
-              alertText = MessageFormat.format(HEALTHY_MSG, hostName);
-              break;
-            case WAITING_FOR_HOST_STATUS_UPDATES:
-              alertText = MessageFormat.format(STATUS_UPDATE_MSG, hostName);
-              break;
-            case HEARTBEAT_LOST:
-              alertState = AlertState.CRITICAL;
-              alertText = MessageFormat.format(HEARTBEAT_LOST_MSG, hostName);
-              break;
-            case UNHEALTHY:
-              alertState = AlertState.CRITICAL;
-              alertText = MessageFormat.format(UNHEALTHY_MSG, hostName);
-            default:
-              alertState = AlertState.UNKNOWN;
-              alertText = MessageFormat.format(UNKNOWN_MSG, hostName, hostState);
-              break;
-          }
-
-          Alert alert = new Alert(entity.getDefinitionName(), null,
-              entity.getServiceName(), entity.getComponentName(), hostName,
-              alertState);
-
-          alert.setLabel(entity.getLabel());
-          alert.setText(alertText);
-          alert.setTimestamp(alertTimestamp);
-          alert.setCluster(cluster.getClusterName());
-
-          AlertReceivedEvent event = new AlertReceivedEvent(
-              cluster.getClusterId(), alert);
-
-          m_alertEventPublisher.publish(event);
-        }
-      }
-    } catch (Exception exception) {
-      LOG.error("Unable to run the {} alert", HEARTBEAT_DEFINITION_NAME,
-          exception);
+      alerts.add(alert);
     }
+
+    return alerts;
   }
 }

ambari-server/src/main/java/org/apache/ambari/server/alerts/AlertRunnable.java

@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.alerts;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.events.AlertEvent;
+import org.apache.ambari.server.events.AlertReceivedEvent;
+import org.apache.ambari.server.events.publishers.AlertEventPublisher;
+import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
+import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
+import org.apache.ambari.server.state.Alert;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+
+/**
+ * The {@link AlertRunnable} class is used to boilerplate the expected
+ * functionality of {@link Runnable}s which need to eventually create and fire
+ * {@link AlertReceivedEvent}s.
+ * <p/>
+ * Implementations of this class do not need to concern themselves with checking
+ * for whether their alert definition is enabled or with constructing and firing
+ * the alert events.
+ */
+public abstract class AlertRunnable implements Runnable {
+
+  /**
+   * Logger.
+   */
+  private final static Logger LOG = LoggerFactory.getLogger(AlertRunnable.class);
+
+  /**
+   * The alert definition name.
+   */
+  protected final String m_definitionName;
+
+  /**
+   * Used to get alert definitions to use when generating alert instances.
+   */
+  @Inject
+  private Provider<Clusters> m_clustersProvider;
+
+  /**
+   * Used for looking up alert definitions.
+   */
+  @Inject
+  private AlertDefinitionDAO m_dao;
+
+  /**
+   * Publishes {@link AlertEvent} instances.
+   */
+  @Inject
+  private AlertEventPublisher m_alertEventPublisher;
+
+  /**
+   * Constructor.
+   *
+   * @param definitionName
+   *          the definition name (not {@code null}).
+   */
+  public AlertRunnable(String definitionName) {
+    m_definitionName = definitionName;
+  }
+
+  /**
+   * Invoked on subclasses when it is time for them to check their specific
+   * alert criteria. Implementations must return a list of {@link Alert}
+   * instances which will be turned into {@link AlertEvent}s and published.
+   * <p/>
+   * This method will only be invoked if the definition is enabled for the
+   * specified cluster.
+   *
+   * @param cluster
+   *          the cluster for which the alert is executing. If there are
+   *          multiple clusters defined, then this method is called once for
+   *          each.
+   * @param definition
+   *          the information about this concrete alert.
+   * @return a list of {@link Alert} to be used to create {@link AlertEvent}s
+   *         and published with the {@link AlertEventPublisher} (not
+   *         {@code null}).
+   * @throws AmbariException
+   */
+  abstract List<Alert> execute(Cluster cluster, AlertDefinitionEntity definition)
+      throws AmbariException;
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public final void run() {
+    try {
+      Map<String, Cluster> clusterMap = m_clustersProvider.get().getClusters();
+      for (Cluster cluster : clusterMap.values()) {
+        AlertDefinitionEntity definition = m_dao.findByName(cluster.getClusterId(), m_definitionName);
+
+        // skip this cluster if the runnable's alert definition is missing or
+        // disabled
+        if (null == definition || !definition.getEnabled()) {
+          continue;
+        }
+
+        // for every alert generated from the implementation, fire an event
+        List<Alert> alerts = execute(cluster, definition);
+        for (Alert alert : alerts) {
+          AlertReceivedEvent event = new AlertReceivedEvent(cluster.getClusterId(), alert);
+          m_alertEventPublisher.publish(event);
+        }
+      }
+    } catch (Exception exception) {
+      LOG.error("Unable to run the {} alert", m_definitionName, exception);
+    }
+  }
+}

ambari-server/src/main/java/org/apache/ambari/server/alerts/AmbariPerformanceRunnable.java

@@ -0,0 +1,435 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.alerts;
+
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.actionmanager.ActionManager;
+import org.apache.ambari.server.actionmanager.RequestStatus;
+import org.apache.ambari.server.api.services.BaseRequest;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.internal.AbstractControllerResourceProvider;
+import org.apache.ambari.server.controller.internal.ClusterResourceProvider;
+import org.apache.ambari.server.controller.spi.Predicate;
+import org.apache.ambari.server.controller.spi.Request;
+import org.apache.ambari.server.controller.spi.Resource.Type;
+import org.apache.ambari.server.controller.spi.ResourceProvider;
+import org.apache.ambari.server.controller.utilities.PredicateBuilder;
+import org.apache.ambari.server.controller.utilities.PropertyHelper;
+import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
+import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
+import org.apache.ambari.server.state.Alert;
+import org.apache.ambari.server.state.AlertState;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.alert.AlertDefinition;
+import org.apache.ambari.server.state.alert.AlertDefinitionFactory;
+import org.apache.ambari.server.state.alert.ParameterizedSource.AlertParameter;
+import org.apache.ambari.server.state.alert.ServerSource;
+import org.apache.ambari.server.state.services.AmbariServerAlertService;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.math.NumberUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+
+/**
+ * The {@link AmbariPerformanceRunnable} is used by the
+ * {@link AmbariServerAlertService} to ensure that certain areas of Ambari are
+ * responsive. It performs the following checks:
+ * <ul>
+ * <li>A GET request against the cluster endpoint.</li>
+ * <li>A query against {@link HostRoleCommandDAO} to get a summary of request
+ * statuses</li>
+ * <ul>
+ */
+public class AmbariPerformanceRunnable extends AlertRunnable {
+
+  /**
+   * Logger.
+   */
+  private final static Logger LOG = LoggerFactory.getLogger(AmbariPerformanceRunnable.class);
+
+  /**
+   * <pre>
+   * Performance Overview:
+   *   Database Access (Request By Status): 330ms (OK)
+   *   REST API (Cluster Request): 5,456ms (WARNING)
+   * </pre>
+   */
+  private static final String PERFORMANCE_OVERVIEW_TEMPLATE = "Performance Overview:"
+      + System.lineSeparator() + "{0}";
+
+  /**
+   * Example: {@code Database Access (Request By Status): 330ms (OK)}
+   */
+  private static final String PERFORMANCE_AREA_TEMPLATE = "  {0}: {1}ms ({2})";
+
+  /**
+   * Example:
+   * {@code Unable to execute performance alert area REQUEST_BY_STATUS (UNKNOWN)}
+   */
+  private static final String PERFORMANCE_AREA_FAILURE_TEMPLATE = "  Unable to execute performance alert area {0}: ({1})";
+
+  /**
+   * Used for converting {@link AlertDefinitionEntity} into
+   * {@link AlertDefinition} instances.
+   */
+  @Inject
+  private AlertDefinitionFactory m_definitionFactory;
+
+  /**
+   * The {@link PerformanceArea} enumeration represents logical areas of
+   * functionality to test for performance.
+   */
+  private enum PerformanceArea {
+
+    /**
+     * Query for requests by {@link RequestStatus#IN_PROGRESS}.
+     */
+    REQUEST_BY_STATUS("Database Access (Request By Status)",
+        "request.by.status.warning.threshold", 3000, "request.by.status.critical.threshold", 5000) {
+      /**
+       * {@inheritDoc}
+       */
+      @Override
+      void execute(AmbariPerformanceRunnable runnable, Cluster cluster) throws Exception {
+        runnable.m_actionManager.get().getRequestsByStatus(RequestStatus.IN_PROGRESS,
+            BaseRequest.DEFAULT_PAGE_SIZE, false);
+      }
+    },
+
+    /**
+     * Query through the REST API framework for a cluster.
+     */
+    REST_API_GET_CLUSTER("REST API (Cluster)",
+        "rest.api.cluster.warning.threshold",
+        5000, "rest.api.cluster.critical.threshold", 7000) {
+      /**
+       * {@inheritDoc}
+       */
+      @Override
+      void execute(AmbariPerformanceRunnable runnable, Cluster cluster) throws Exception {
+        Type type = Type.Cluster;
+        ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider(
+            type, PropertyHelper.getPropertyIds(type), PropertyHelper.getKeyPropertyIds(type),
+            runnable.m_amc.get());
+
+        Set<String> propertyIds = new HashSet<String>();
+
+        propertyIds.add(ClusterResourceProvider.CLUSTER_ID_PROPERTY_ID);
+        propertyIds.add(ClusterResourceProvider.CLUSTER_NAME_PROPERTY_ID);
+
+        // create the request
+        Request request = PropertyHelper.getReadRequest(propertyIds);
+
+        // build the predicate for this cluster
+        Predicate predicate = new PredicateBuilder().property(
+            ClusterResourceProvider.CLUSTER_NAME_PROPERTY_ID).equals(
+                cluster.getClusterName()).toPredicate();
+
+        provider.getResources(request, predicate);
+      }
+    };
+
+    /**
+     * The label for the performance area.
+     */
+    private final String m_label;
+
+    /**
+     * The name of the parameter on the alert definition which represents the
+     * {@link AlertState#WARNING} threshold value.
+     */
+    private final String m_warningParameter;
+
+    /**
+     * A default {@link AlertState#WARNING} threshold value of the definition
+     * doesn't have {@link #m_warningParameter} defined.
+     */
+    private final int m_defaultWarningThreshold;
+
+    /**
+     * The name of the parameter on the alert definition which represents the
+     * {@link AlertState#CRITICAL} threshold value.
+     */
+    private final String m_criticalParameter;
+
+    /**
+     * A default {@link AlertState#WARNING} threshold value of the definition
+     * doesn't have {@link #m_criticalParameter} defined.
+     */
+    private final int m_defaultCriticalThreshold;
+
+    /**
+     * Constructor.
+     *
+     * @param label
+     *          the display label for this performance area (not {@code null}).
+     * @param warningParameter
+     *          the definition parameter name for the warning threshold (not
+     *          {@code null})
+     * @param defaultWarningThreshold
+     *          the default value to use if the definition does not have a
+     *          warning threshold paramter.
+     * @param criticalParameter
+     *          the definition parameter name for the critical threshold (not
+     *          {@code null})
+     * @param defaultCriticalThreshold
+     *          the default value to use if the definition does not have a
+     *          critical threshold paramter.
+     */
+    private PerformanceArea(String label, String warningParameter, int defaultWarningThreshold,
+        String criticalParameter, int defaultCriticalThreshold) {
+      m_label = label;
+      m_warningParameter = warningParameter;
+      m_defaultWarningThreshold = defaultWarningThreshold;
+      m_criticalParameter = criticalParameter;
+      m_defaultCriticalThreshold = defaultCriticalThreshold;
+    }
+
+    /**
+     * Runs the {@link PerformanceArea}.
+     *
+     * @param runnable
+     *          a reference to the parent {@link AlertRunnable} which has
+     *          injected members for use.
+     * @return a result of running the performance area (never {@code null}).
+     */
+    abstract void execute(AmbariPerformanceRunnable runnable, Cluster cluster) throws Exception;
+  }
+
+  /**
+   * Used for querying for requests by status.
+   */
+  @Inject
+  private Provider<ActionManager> m_actionManager;
+
+  @Inject
+  private Provider<AmbariManagementController> m_amc;
+
+  /**
+   * Constructor.
+   *
+   * @param definitionName
+   */
+  public AmbariPerformanceRunnable(String definitionName) {
+    super(definitionName);
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  List<Alert> execute(Cluster cluster, AlertDefinitionEntity entity) throws AmbariException {
+    // coerce the entity into a business object so that the list of parameters
+    // can be extracted and used for threshold calculation
+    AlertDefinition definition = m_definitionFactory.coerce(entity);
+    ServerSource serverSource = (ServerSource) definition.getSource();
+    List<AlertParameter> parameters = serverSource.getParameters();
+    List<String> results = new ArrayList<>();
+
+    // start out assuming OK
+    AlertState alertState = AlertState.OK;
+
+    // run every performance area
+    for (PerformanceArea performanceArea : PerformanceArea.values()) {
+      // execute the performance area, creating an UNKNOWN state on exceptions
+      PerformanceResult performanceResult;
+      try {
+        long startTime = System.currentTimeMillis();
+        performanceArea.execute(this, cluster);
+        long totalTime = System.currentTimeMillis() - startTime;
+
+        performanceResult = calculatePerformanceResult(performanceArea, totalTime, parameters);
+
+      } catch (Exception exception) {
+        String result = MessageFormat.format(PERFORMANCE_AREA_FAILURE_TEMPLATE, performanceArea,
+            AlertState.UNKNOWN);
+
+        LOG.error(result, exception);
+        performanceResult = new PerformanceResult(result, AlertState.UNKNOWN);
+      }
+
+      String result = performanceResult.getResult();
+      AlertState resultAlertState = performanceResult.getAlertState();
+
+      // keep track of the string result for formatting later
+      results.add(result);
+
+      // keep track of the overall state of "this" alert
+      switch (resultAlertState) {
+        case CRITICAL:
+          alertState = AlertState.CRITICAL;
+          break;
+        case OK:
+          break;
+        case SKIPPED:
+          break;
+        case UNKNOWN:
+          if (alertState == AlertState.OK) {
+            alertState = AlertState.UNKNOWN;
+          }
+          break;
+        case WARNING:
+          if (alertState != AlertState.CRITICAL) {
+            alertState = AlertState.WARNING;
+          }
+          break;
+        default:
+          break;
+      }
+    }
+
+    // create a text overview of all of the runs
+    String allResults = StringUtils.join(results, System.lineSeparator());
+    String overview = MessageFormat.format(PERFORMANCE_OVERVIEW_TEMPLATE, allResults);
+
+    // build the alert to return
+    Alert alert = new Alert(entity.getDefinitionName(), null, entity.getServiceName(),
+        entity.getComponentName(), null, alertState);
+
+    alert.setLabel(entity.getLabel());
+    alert.setText(overview);
+    alert.setTimestamp(System.currentTimeMillis());
+    alert.setCluster(cluster.getClusterName());
+
+    return Collections.singletonList(alert);
+  }
+
+  /**
+   * Calculates the state based on the threshold values for a
+   * {@link PerformanceArea} and an actual run time.
+   *
+   * @param area
+   *          the area to calculate the result for (not {@code null}).
+   * @param time
+   *          the time taken, in milliseconds, to run the test.
+   * @param parameters
+   *          a list of parameters from the alert definition which contain the
+   *          threshold values.
+   * @return a result of running the performance area (never {@code null}).
+   */
+  PerformanceResult calculatePerformanceResult(PerformanceArea area, long time,
+      List<AlertParameter> parameters) {
+    AlertState alertState = AlertState.OK;
+    int warningThreshold = area.m_defaultWarningThreshold;
+    int criticalThreshold = area.m_defaultCriticalThreshold;
+
+    for (AlertParameter parameter : parameters) {
+      Object value = parameter.getValue();
+
+      if (StringUtils.equals(parameter.getName(), area.m_warningParameter)) {
+        warningThreshold = getThresholdValue(value, warningThreshold);
+      }
+
+      if (StringUtils.equals(parameter.getName(), area.m_criticalParameter)) {
+        criticalThreshold = getThresholdValue(value, criticalThreshold);
+      }
+    }
+
+    if (time >= warningThreshold && time < criticalThreshold) {
+      alertState = AlertState.WARNING;
+    }
+
+    if (time >= criticalThreshold) {
+      alertState = AlertState.WARNING;
+    }
+
+    String resultLabel = MessageFormat.format(PERFORMANCE_AREA_TEMPLATE, area.m_label, time,
+        alertState);
+
+    return new PerformanceResult(resultLabel, alertState);
+  }
+
+  /**
+   * Converts the given value to an integer safely.
+   *
+   * @param value
+   * @param defaultValue
+   * @return
+   */
+  int getThresholdValue(Object value, int defaultValue) {
+    if (null == value) {
+      return defaultValue;
+    }
+
+    if (value instanceof Number) {
+      return ((Number) value).intValue();
+    }
+
+    if (!(value instanceof String)) {
+      value = value.toString();
+    }
+
+    if (!NumberUtils.isNumber((String) value)) {
+      return defaultValue;
+    }
+
+    Number number = NumberUtils.createNumber((String) value);
+    return number.intValue();
+  }
+
+  /**
+   * The {@link PerformanceResult} class is used to wrap the result of a
+   * {@link PerformanceArea}.
+   */
+  private static final class PerformanceResult {
+    private final String m_result;
+    private final AlertState m_alertState;
+
+    /**
+     * Constructor.
+     *
+     * @param result
+     *          the text of the result (not {@code null}).
+     * @param alertState
+     *          the result state (not {@code null}).
+     */
+    private PerformanceResult(String result, AlertState alertState) {
+      m_result = result;
+      m_alertState = alertState;
+    }
+
+    /**
+     * Gets the fully-rendered result text, such as:
+     * {@code Database Access (Request By Status): 330ms (OK)}
+     *
+     * @return the result
+     */
+    public String getResult() {
+      return m_result;
+    }
+
+    /**
+     * The state of the result as calculated by the threshold parameters.
+     *
+     * @return the state
+     */
+    public AlertState getAlertState() {
+      return m_alertState;
+    }
+  }
+}

ambari-server/src/main/java/org/apache/ambari/server/alerts/StaleAlertRunnable.java

@@ -18,16 +18,13 @@
 package org.apache.ambari.server.alerts;
 
 import java.text.MessageFormat;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 
-import org.apache.ambari.server.events.AlertEvent;
-import org.apache.ambari.server.events.AlertReceivedEvent;
-import org.apache.ambari.server.events.publishers.AlertEventPublisher;
-import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
 import org.apache.ambari.server.orm.dao.AlertsDAO;
 import org.apache.ambari.server.orm.entities.AlertCurrentEntity;
 import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
@@ -35,16 +32,12 @@ import org.apache.ambari.server.orm.entities.AlertHistoryEntity;
 import org.apache.ambari.server.state.Alert;
 import org.apache.ambari.server.state.AlertState;
 import org.apache.ambari.server.state.Cluster;
-import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.MaintenanceState;
 import org.apache.ambari.server.state.alert.SourceType;
 import org.apache.ambari.server.state.services.AmbariServerAlertService;
 import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import com.google.inject.Inject;
-import com.google.inject.Provider;
 
 /**
  * The {@link StaleAlertRunnable} is used by the
@@ -53,18 +46,7 @@ import com.google.inject.Provider;
  * single alert with {@link AlertState#CRITICAL} along with a textual
  * description of the alerts that are stale.
  */
-public class StaleAlertRunnable implements Runnable {
-
-  /**
-   * Logger.
-   */
-  private final static Logger LOG = LoggerFactory.getLogger(StaleAlertRunnable.class);
-
-  /**
-   * The unique name for the alert definition that governs this service.
-   */
-  private static final String STALE_ALERT_DEFINITION_NAME = "ambari_server_stale_alerts";
-
+public class StaleAlertRunnable extends AlertRunnable {
   /**
    * The message for the alert when all services have run in their designated
    * intervals.
@@ -85,6 +67,10 @@ public class StaleAlertRunnable implements Runnable {
    */
   private static final long MINUTE_TO_MS_CONVERSION = 60L * 1000L;
 
+  private static final long MILLISECONDS_PER_MINUTE = 1000L * 60L;
+  private static final int MINUTES_PER_DAY = 24 * 60;
+  private static final int MINUTES_PER_HOUR = 60;
+
   /**
    * Used to get the current alerts and the last time they ran.
    */
@@ -92,154 +78,113 @@ public class StaleAlertRunnable implements Runnable {
   private AlertsDAO m_alertsDao;
 
   /**
-   * Used to get alert definitions to use when generating alert instances.
-   */
-  @Inject
-  private Provider<Clusters> m_clustersProvider;
-
-  /**
-   * Used for looking up alert definitions.
-   */
-  @Inject
-  private AlertDefinitionDAO m_dao;
-
-  /**
-   * Publishes {@link AlertEvent} instances.
+   * Constructor.
+   *
+   * @param definitionName
    */
-  @Inject
-  private AlertEventPublisher m_alertEventPublisher;
-
-  /**
-   * Constructor. Required for type introspection by
-   * {@link AmbariServerAlertService}.
-   */
-  public StaleAlertRunnable() {
+  public StaleAlertRunnable(String definitionName) {
+    super(definitionName);
   }
 
   /**
    * {@inheritDoc}
    */
   @Override
-  public void run() {
-    try {
-      Map<String, Cluster> clusterMap = m_clustersProvider.get().getClusters();
-      for (Cluster cluster : clusterMap.values()) {
-        AlertDefinitionEntity entity = m_dao.findByName(cluster.getClusterId(),
-            STALE_ALERT_DEFINITION_NAME);
-
-        // skip this cluster if the runnable's alert definition is missing or
-        // disabled
-        if (null == entity || !entity.getEnabled()) {
-          continue;
-        }
+  List<Alert> execute(Cluster cluster, AlertDefinitionEntity myDefinition) {
+    Set<String> staleAlerts = new TreeSet<String>();
+    Map<String, Set<String>> staleHostAlerts = new HashMap<>();
+    Set<String> hostsWithStaleAlerts = new TreeSet<>();
 
-        long now = System.currentTimeMillis();
-        Set<String> staleAlerts = new TreeSet<String>();
-        Map<String, Set<String>> staleHostAlerts = new HashMap<>();
-        Set<String> hostsWithStaleAlerts = new TreeSet<>();
+    // get the cluster's current alerts
+    List<AlertCurrentEntity> currentAlerts = m_alertsDao.findCurrentByCluster(
+        cluster.getClusterId());
 
-        // get the cluster's current alerts
-        List<AlertCurrentEntity> currentAlerts = m_alertsDao.findCurrentByCluster(cluster.getClusterId());
+    long now = System.currentTimeMillis();
 
-        // for each current alert, check to see if the last time it ran is
-        // more than 2x its interval value (indicating it hasn't run)
-        for (AlertCurrentEntity current : currentAlerts) {
-          AlertHistoryEntity history = current.getAlertHistory();
-          AlertDefinitionEntity definition = history.getAlertDefinition();
+    // for each current alert, check to see if the last time it ran is
+    // more than 2x its interval value (indicating it hasn't run)
+    for (AlertCurrentEntity current : currentAlerts) {
+      AlertHistoryEntity history = current.getAlertHistory();
+      AlertDefinitionEntity currentDefinition = history.getAlertDefinition();
 
-          // skip aggregates as they are special
-          if (definition.getSourceType() == SourceType.AGGREGATE) {
-            continue;
-          }
+      // skip aggregates as they are special
+      if (currentDefinition.getSourceType() == SourceType.AGGREGATE) {
+          continue;
+        }
 
-          // skip alerts in maintenance mode
-          if (current.getMaintenanceState() != MaintenanceState.OFF) {
-            continue;
-          }
+      // skip alerts in maintenance mode
+      if (current.getMaintenanceState() != MaintenanceState.OFF) {
+        continue;
+      }
 
-          // skip alerts that have not run yet
-          if (current.getLatestTimestamp() == 0) {
-            continue;
-          }
+      // skip alerts that have not run yet
+      if (current.getLatestTimestamp() == 0) {
+        continue;
+      }
 
-          // skip this alert (who watches the watchers)
-          if (definition.getDefinitionName().equals(STALE_ALERT_DEFINITION_NAME)) {
-            continue;
-          }
+      // skip this alert (who watches the watchers)
+      if (currentDefinition.getDefinitionName().equals(m_definitionName)) {
+        continue;
+      }
 
-          // convert minutes to milliseconds for the definition's interval
-          long intervalInMillis = definition.getScheduleInterval()
-              * MINUTE_TO_MS_CONVERSION;
+      // convert minutes to milliseconds for the definition's interval
+      long intervalInMillis = currentDefinition.getScheduleInterval() * MINUTE_TO_MS_CONVERSION;
 
-          // if the last time it was run is >= 2x the interval, it's stale
-          long timeDifference = now - current.getLatestTimestamp();
-          if (timeDifference >= 2 * intervalInMillis) {
+      // if the last time it was run is >= 2x the interval, it's stale
+      long timeDifference = now - current.getLatestTimestamp();
+      if (timeDifference >= 2 * intervalInMillis) {
 
-            // it is technically possible to have a null/blank label; if so,
-            // default to the name of the definition
-            String label = definition.getLabel();
-            if (StringUtils.isEmpty(label)) {
-              label = definition.getDefinitionName();
-            }
+        // it is technically possible to have a null/blank label; if so,
+        // default to the name of the definition
+        String label = currentDefinition.getLabel();
+        if (StringUtils.isEmpty(label)) {
+          label = currentDefinition.getDefinitionName();
+          }
 
-            if (null != history.getHostName()) {
-              // keep track of the host, if not null
-              String hostName = history.getHostName();
-              hostsWithStaleAlerts.add(hostName);
-              if(!staleHostAlerts.containsKey(hostName)) {
-                staleHostAlerts.put(hostName, new TreeSet<String>());
-              }
-
-              staleHostAlerts.get(hostName).add(MessageFormat.format(TIMED_LABEL_MSG, label,
-                  millisToHumanReadableStr(timeDifference)));
-            } else {
-              // non host alerts
-              staleAlerts.add(label);
+        if (null != history.getHostName()) {
+          // keep track of the host, if not null
+          String hostName = history.getHostName();
+          hostsWithStaleAlerts.add(hostName);
+          if (!staleHostAlerts.containsKey(hostName)) {
+            staleHostAlerts.put(hostName, new TreeSet<String>());
             }
-          }
-        }
 
-        for(String host : staleHostAlerts.keySet()) {
-          staleAlerts.add(MessageFormat.format(HOST_LABEL_MSG, host,
-              StringUtils.join(staleHostAlerts.get(host), ",\n  ")));
+          staleHostAlerts.get(hostName).add(MessageFormat.format(TIMED_LABEL_MSG, label,
+              millisToHumanReadableStr(timeDifference)));
+        } else {
+          // non host alerts
+          staleAlerts.add(label);
+          }
         }
+    }
 
-        AlertState alertState = AlertState.OK;
-        String alertText = ALL_ALERTS_CURRENT_MSG;
+    for (String host : staleHostAlerts.keySet()) {
+      staleAlerts.add(MessageFormat.format(HOST_LABEL_MSG, host,
+          StringUtils.join(staleHostAlerts.get(host), ",\n  ")));
+    }
 
-        // if there are stale alerts, mark as CRITICAL with the list of
-        // alerts
-        if( !staleAlerts.isEmpty() ){
-          alertState = AlertState.CRITICAL;
-          alertText = MessageFormat.format(STALE_ALERTS_MSG,
-              staleAlerts.size(), hostsWithStaleAlerts.size(),
-              StringUtils.join(staleAlerts, ",\n"));
-        }
+    AlertState alertState = AlertState.OK;
+    String alertText = ALL_ALERTS_CURRENT_MSG;
 
-        Alert alert = new Alert(entity.getDefinitionName(), null,
-            entity.getServiceName(), entity.getComponentName(), null,
-            alertState);
+    // if there are stale alerts, mark as CRITICAL with the list of
+    // alerts
+    if (!staleAlerts.isEmpty()) {
+      alertState = AlertState.CRITICAL;
+      alertText = MessageFormat.format(STALE_ALERTS_MSG, staleAlerts.size(),
+          hostsWithStaleAlerts.size(), StringUtils.join(staleAlerts, ",\n"));
+    }
 
-        alert.setLabel(entity.getLabel());
-        alert.setText(alertText);
-        alert.setTimestamp(now);
-        alert.setCluster(cluster.getClusterName());
+    Alert alert = new Alert(myDefinition.getDefinitionName(), null, myDefinition.getServiceName(),
+        myDefinition.getComponentName(), null, alertState);
 
-        AlertReceivedEvent event = new AlertReceivedEvent(
-            cluster.getClusterId(), alert);
+    alert.setLabel(myDefinition.getLabel());
+    alert.setText(alertText);
+    alert.setTimestamp(now);
+    alert.setCluster(cluster.getClusterName());
 
-        m_alertEventPublisher.publish(event);
-      }
-    } catch (Exception exception) {
-      LOG.error("Unable to run the {} alert", STALE_ALERT_DEFINITION_NAME,
-          exception);
-    }
+    return Collections.singletonList(alert);
   }
 
-  private static final long MILLISECONDS_PER_MINUTE = 1000l * 60l;
-  private static final int MINUTES_PER_DAY = 24 * 60;
-  private static final int MINUTES_PER_HOUR = 60;
-
   /**
    * Converts given {@code milliseconds} to human-readable {@link String} like "1d 2h 3m" or "2h 4m".
    * @param milliseconds milliseconds to convert

ambari-server/src/main/java/org/apache/ambari/server/state/alert/ParameterizedSource.java

@@ -0,0 +1,264 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.state.alert;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.ambari.server.state.AlertState;
+
+import com.google.gson.annotations.SerializedName;
+
+
+/**
+ * The {@link ParameterizedSource} is used for alerts where the logic of
+ * computing the {@link AlertState} is dependant on user-specified parameters.
+ * For example, the parameters might be threshold values.
+ */
+public abstract class ParameterizedSource extends Source {
+
+  /**
+   * A list of all of the alert parameters, if any.
+   */
+  @SerializedName("parameters")
+  List<AlertParameter> m_parameters;
+
+  /**
+   * Gets a list of the optional parameters which govern how a parameterized
+   * alert behaves. These are usually threshold values.
+   *
+   * @return the list of parameters, or an empty list if none.
+   */
+  public List<AlertParameter> getParameters() {
+    if (null == m_parameters) {
+      return Collections.emptyList();
+    }
+
+    return m_parameters;
+  }
+
+  /**
+   * The {@link AlertParameter} class represents a single parameter that can be
+   * passed into an alert which takes parameters.
+   */
+  public static class AlertParameter {
+    @SerializedName("name")
+    private String m_name;
+
+    @SerializedName("display_name")
+    private String m_displayName;
+
+    @SerializedName("units")
+    private String m_units;
+
+    @SerializedName("value")
+    private Object m_value;
+
+    @SerializedName("description")
+    private String m_description;
+
+    @SerializedName("type")
+    private AlertParameterType m_type;
+
+    @SerializedName("visibility")
+    private AlertParameterVisibility m_visibility;
+
+    /**
+     * If this alert parameter controls a threshold, then its specified here,
+     * otherwise it's {@code null}.
+     */
+    @SerializedName("threshold")
+    private AlertState m_threshold;
+
+    /**
+     * Gets the unique name of the parameter.
+     *
+     * @return the name
+     */
+    public String getName() {
+      return m_name;
+    }
+
+    /**
+     * Gets the human readable name of the parameter.
+     *
+     * @return the displayName
+     */
+    public String getDisplayName() {
+      return m_displayName;
+    }
+
+    /**
+     * Gets the display units of the paramter.
+     *
+     * @return the units
+     */
+    public String getUnits() {
+      return m_units;
+    }
+
+    /**
+     * Gets the value of the parameter.
+     *
+     * @return the value
+     */
+    public Object getValue() {
+      return m_value;
+    }
+
+    /**
+     * Gets the description of the parameter.
+     *
+     * @return the description
+     */
+    public String getDescription() {
+      return m_description;
+    }
+
+    /**
+     * Gets the visibility of the parameter.
+     *
+     * @return the visibility
+     */
+    public AlertParameterVisibility getVisibility() {
+      return m_visibility;
+    }
+
+    /**
+     * Gets the threshold that this parameter directly controls, or {@code null}
+     * for none.
+     *
+     * @return the threshold, or {@code null}.
+     */
+    public AlertState getThreshold() {
+      return m_threshold;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public int hashCode() {
+      final int prime = 31;
+      int result = 1;
+      result = prime * result + ((m_description == null) ? 0 : m_description.hashCode());
+      result = prime * result + ((m_displayName == null) ? 0 : m_displayName.hashCode());
+      result = prime * result + ((m_name == null) ? 0 : m_name.hashCode());
+      result = prime * result + ((m_threshold == null) ? 0 : m_threshold.hashCode());
+      result = prime * result + ((m_type == null) ? 0 : m_type.hashCode());
+      result = prime * result + ((m_units == null) ? 0 : m_units.hashCode());
+      result = prime * result + ((m_value == null) ? 0 : m_value.hashCode());
+      result = prime * result + ((m_visibility == null) ? 0 : m_visibility.hashCode());
+      return result;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public boolean equals(Object obj) {
+      if (this == obj) {
+        return true;
+      }
+      if (obj == null) {
+        return false;
+      }
+      if (getClass() != obj.getClass()) {
+        return false;
+      }
+      AlertParameter other = (AlertParameter) obj;
+      if (m_description == null) {
+        if (other.m_description != null) {
+          return false;
+        }
+      } else if (!m_description.equals(other.m_description)) {
+        return false;
+      }
+      if (m_displayName == null) {
+        if (other.m_displayName != null) {
+          return false;
+        }
+      } else if (!m_displayName.equals(other.m_displayName)) {
+        return false;
+      }
+      if (m_name == null) {
+        if (other.m_name != null) {
+          return false;
+        }
+      } else if (!m_name.equals(other.m_name)) {
+        return false;
+      }
+      if (m_threshold != other.m_threshold) {
+        return false;
+      }
+      if (m_type != other.m_type) {
+        return false;
+      }
+      if (m_units == null) {
+        if (other.m_units != null) {
+          return false;
+        }
+      } else if (!m_units.equals(other.m_units)) {
+        return false;
+      }
+      if (m_value == null) {
+        if (other.m_value != null) {
+          return false;
+        }
+      } else if (!m_value.equals(other.m_value)) {
+        return false;
+      }
+      if (m_visibility == null) {
+        if (other.m_visibility != null) {
+          return false;
+        }
+      } else if (!m_visibility.equals(other.m_visibility)) {
+        return false;
+      }
+      return true;
+    }
+  }
+
+  /**
+   * The {@link AlertParameterType} enum represents the value type.
+   */
+  public enum AlertParameterType {
+    /**
+     * String
+     */
+    STRING,
+
+    /**
+     * Integers, longs, floats, etc.
+     */
+    NUMERIC,
+
+    /**
+     * A percent value, expessed as a float.
+     */
+    PERCENT
+  }
+
+  /**
+   * The {@link AlertParameterVisibility} enum represents the visibility of
+   * alert parameters.
+   */
+  public enum AlertParameterVisibility {
+    VISIBLE, HIDDEN, READ_ONLY
+  }
+}

ambari-server/src/main/java/org/apache/ambari/server/state/alert/ScriptSource.java

@@ -17,10 +17,6 @@
  */
 package org.apache.ambari.server.state.alert;
 
-import java.util.List;
-
-import org.apache.ambari.server.state.AlertState;
-
 import com.google.gson.annotations.SerializedName;
 
 /**
@@ -29,17 +25,11 @@ import com.google.gson.annotations.SerializedName;
  * Equality checking for instances of this class should be executed on every
  * member to ensure that reconciling stack differences is correct.
  */
-public class ScriptSource extends Source {
+public class ScriptSource extends ParameterizedSource {
 
   @SerializedName("path")
   private String m_path = null;
 
-  /**
-   * A list of all of the script parameters, if any.
-   */
-  @SerializedName("parameters")
-  private List<ScriptParameter> m_parameters;
-
   /**
    * @return the path to the script file.
    */
@@ -88,216 +78,4 @@ public class ScriptSource extends Source {
 
     return true;
   }
-
-  /**
-   * The {@link ScriptParameter} class represents a single parameter that can be
-   * passed into a script alert.
-   */
-  public static class ScriptParameter {
-    @SerializedName("name")
-    private String m_name;
-
-    @SerializedName("display_name")
-    private String m_displayName;
-
-    @SerializedName("units")
-    private String m_units;
-
-    @SerializedName("value")
-    private Object m_value;
-
-    @SerializedName("description")
-    private String m_description;
-
-    @SerializedName("type")
-    private ScriptParameterType m_type;
-
-
-    @SerializedName("visibility")
-    private ScriptParameterVisibility m_visibility;
-
-    /**
-     * If this script parameter controls a threshold, then its specified here,
-     * otherwise it's {@code null}.
-     */
-    @SerializedName("threshold")
-    private AlertState m_threshold;
-
-    /**
-     * Gets the unique name of the parameter.
-     *
-     * @return the name
-     */
-    public String getName() {
-      return m_name;
-    }
-
-    /**
-     * Gets the human readable name of the parameter.
-     *
-     * @return the displayName
-     */
-    public String getDisplayName() {
-      return m_displayName;
-    }
-
-    /**
-     * Gets the display units of the paramter.
-     *
-     * @return the units
-     */
-    public String getUnits() {
-      return m_units;
-    }
-
-    /**
-     * Gets the value of the parameter.
-     *
-     * @return the value
-     */
-    public Object getValue() {
-      return m_value;
-    }
-
-    /**
-     * Gets the description of the parameter.
-     *
-     * @return the description
-     */
-    public String getDescription() {
-      return m_description;
-    }
-
-    /**
-     * Gets the visibility of the parameter.
-     * @return the visibility
-     */
-    public ScriptParameterVisibility getVisibility() {
-      return m_visibility;
-    }
-
-    /**
-     * Gets the threshold that this parameter directly controls, or {@code null}
-     * for none.
-     *
-     * @return the threshold, or {@code null}.
-     */
-    public AlertState getThreshold() {
-      return m_threshold;
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public int hashCode() {
-      final int prime = 31;
-      int result = 1;
-      result = prime * result + ((m_description == null) ? 0 : m_description.hashCode());
-      result = prime * result + ((m_displayName == null) ? 0 : m_displayName.hashCode());
-      result = prime * result + ((m_name == null) ? 0 : m_name.hashCode());
-      result = prime * result + ((m_threshold == null) ? 0 : m_threshold.hashCode());
-      result = prime * result + ((m_type == null) ? 0 : m_type.hashCode());
-      result = prime * result + ((m_units == null) ? 0 : m_units.hashCode());
-      result = prime * result + ((m_value == null) ? 0 : m_value.hashCode());
-      result = prime * result + ((m_visibility == null) ? 0 : m_visibility.hashCode());
-      return result;
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public boolean equals(Object obj) {
-      if (this == obj) {
-        return true;
-      }
-      if (obj == null) {
-        return false;
-      }
-      if (getClass() != obj.getClass()) {
-        return false;
-      }
-      ScriptParameter other = (ScriptParameter) obj;
-      if (m_description == null) {
-        if (other.m_description != null) {
-          return false;
-        }
-      } else if (!m_description.equals(other.m_description)) {
-        return false;
-      }
-      if (m_displayName == null) {
-        if (other.m_displayName != null) {
-          return false;
-        }
-      } else if (!m_displayName.equals(other.m_displayName)) {
-        return false;
-      }
-      if (m_name == null) {
-        if (other.m_name != null) {
-          return false;
-        }
-      } else if (!m_name.equals(other.m_name)) {
-        return false;
-      }
-      if (m_threshold != other.m_threshold) {
-        return false;
-      }
-      if (m_type != other.m_type) {
-        return false;
-      }
-      if (m_units == null) {
-        if (other.m_units != null) {
-          return false;
-        }
-      } else if (!m_units.equals(other.m_units)) {
-        return false;
-      }
-      if (m_value == null) {
-        if (other.m_value != null) {
-          return false;
-        }
-      } else if (!m_value.equals(other.m_value)) {
-        return false;
-      }
-      if (m_visibility == null) {
-        if (other.m_visibility != null) {
-          return false;
-        }
-      } else if (!m_visibility.equals(other.m_visibility)) {
-        return false;
-      }
-      return true;
-    }
-
-
-    /**
-     * The {@link ScriptParameterType} enum represents the value type.
-     */
-    public enum ScriptParameterType {
-      /**
-       * String
-       */
-      STRING,
-
-      /**
-       * Integers, longs, floats, etc.
-       */
-      NUMERIC,
-
-      /**
-       * A percent value, expessed as a float.
-       */
-      PERCENT
-    }
-
-    /**
-     * The {@link ScriptParameterVisibility} enum represents the visibility of script parameter.
-     */
-    public enum ScriptParameterVisibility {
-      VISIBLE,
-      HIDDEN,
-      READ_ONLY
-    }
-  }
 }

ambari-server/src/main/java/org/apache/ambari/server/state/alert/ServerSource.java

@@ -23,7 +23,7 @@ import com.google.gson.annotations.SerializedName;
 /**
  * Alert when the source type is defined as {@link SourceType#SERVER}
  */
-public class ServerSource extends Source {
+public class ServerSource extends ParameterizedSource {
 
   @SerializedName("class")
   private String m_class;

ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java

@@ -17,6 +17,8 @@
  */
 package org.apache.ambari.server.state.services;
 
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -26,6 +28,7 @@ import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.ambari.server.AmbariService;
+import org.apache.ambari.server.alerts.AlertRunnable;
 import org.apache.ambari.server.controller.RootServiceResponseFactory.Components;
 import org.apache.ambari.server.controller.RootServiceResponseFactory.Services;
 import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
@@ -46,7 +49,9 @@ import com.google.inject.Provider;
 
 /**
  * The {@link AmbariServerAlertService} is used to manage the dynamically loaded
- * {@link Runnable}s which perform server-side alert checks.
+ * {@link AlertRunnable}s which perform server-side alert checks. The
+ * {@link AlertRunnable}s are scheduled using a {@link ScheduledExecutorService}
+ * with a fixed thread pool size.
  */
 @AmbariService
 public class AmbariServerAlertService extends AbstractScheduledService {
@@ -235,20 +240,24 @@ public class AmbariServerAlertService extends AbstractScheduledService {
 
     try {
       Class<?> clazz = Class.forName(sourceClass);
-      if (!Runnable.class.isAssignableFrom(clazz)) {
+      if (!AlertRunnable.class.isAssignableFrom(clazz)) {
         LOG.warn(
-            "Unable to schedule a server side alert for {} because it is not a Runnable",
-            sourceClass);
+            "Unable to schedule a server side alert for {} because it is not an {}", sourceClass,
+            AlertRunnable.class);
+
         return;
       }
 
       // instantiate and inject
-      Runnable runnable = (Runnable) clazz.newInstance();
-      m_injector.injectMembers(runnable);
+      Constructor<? extends AlertRunnable> constructor = clazz.asSubclass(
+          AlertRunnable.class).getConstructor(String.class);
+
+      AlertRunnable alertRunnable = constructor.newInstance(entity.getDefinitionName());
+      m_injector.injectMembers(alertRunnable);
 
       // schedule the runnable alert
       ScheduledFuture<?> scheduledFuture = m_scheduledExecutorService.scheduleWithFixedDelay(
-          runnable, interval, interval, TimeUnit.MINUTES);
+          alertRunnable, interval, interval, TimeUnit.MINUTES);
 
       String definitionName = entity.getDefinitionName();
       ScheduledAlert scheduledAlert = new ScheduledAlert(scheduledFuture, interval);
@@ -258,9 +267,17 @@ public class AmbariServerAlertService extends AbstractScheduledService {
           definitionName, interval);
 
     } catch (ClassNotFoundException cnfe) {
-      LOG.warn(
+      LOG.error(
           "Unable to schedule a server side alert for {} because it could not be found in the classpath",
           sourceClass);
+    } catch (NoSuchMethodException nsme) {
+      LOG.error(
+          "Unable to schedule a server side alert for {} because it does not have a constructor which takes the proper arguments.",
+          sourceClass);
+    } catch (InvocationTargetException ite) {
+      LOG.error(
+          "Unable to schedule a server side alert for {} because an exception occurred while constructing the instance.",
+          sourceClass, ite);
     }
   }
 

ambari-server/src/main/resources/alerts.json

@@ -26,6 +26,56 @@
           "type": "SERVER",
           "class": "org.apache.ambari.server.alerts.StaleAlertRunnable"
         }
+      },
+      {
+        "name": "ambari_server_performance",
+        "label": "Ambari Server Performance",
+        "description": "This alert is triggered if the server detects that there is a potential performance problem with Ambari. This type of issue can arise for many reasons, but is typically attributed to slow database queries and host resource exhaustion.",
+        "interval": 5,
+        "scope": "SERVICE",
+        "enabled": true,
+        "source": {
+          "type": "SERVER",
+          "class": "org.apache.ambari.server.alerts.AmbariPerformanceRunnable",
+          "parameters": [
+            {
+              "name": "request.by.status.warning.threshold",
+              "display_name": "Warning",
+              "value": 3000,
+              "type": "NUMERIC",
+              "description": "The time to find requests in progress before a warning alert is triggered.",
+              "units": "ms",
+              "threshold": "WARNING"
+            },
+            {
+              "name": "request.by.status.critical.threshold",
+              "display_name": "Critical",
+              "value": 5000,
+              "type": "NUMERIC",
+              "description": "The time to find requests in progress before a critical alert is triggered.",
+              "units": "ms",
+              "threshold": "CRITICAL"
+            },
+            {
+              "name": "rest.api.cluster.warning.threshold",
+              "display_name": "Warning",
+              "value": 5000,
+              "type": "NUMERIC",
+              "description": "The time to get a cluster via the REST API before a warning alert is triggered.",
+              "units": "ms",
+              "threshold": "WARNING"
+            },
+            {
+              "name": "rest.api.cluster.critical.threshold",
+              "display_name": "Critical",
+              "value": 7000,
+              "type": "NUMERIC",
+              "description": "The time to get a cluster via the REST API before a critical alert is triggered.",
+              "units": "ms",
+              "threshold": "CRITICAL"
+            }
+          ]
+        }
       }
     ],
     "AMBARI_AGENT" : [

ambari-server/src/test/java/org/apache/ambari/server/alerts/AgentHeartbeatAlertRunnableTest.java

@@ -25,6 +25,7 @@ import static org.easymock.EasyMock.replay;
 import static org.easymock.EasyMock.verify;
 
 import java.lang.reflect.Field;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -125,10 +126,10 @@ public class AgentHeartbeatAlertRunnableTest {
     // mock the cluster
     expect(m_cluster.getClusterId()).andReturn(CLUSTER_ID).atLeastOnce();
     expect(m_cluster.getClusterName()).andReturn(CLUSTER_NAME).atLeastOnce();
+    expect(m_cluster.getHosts()).andReturn(Collections.singleton(m_host)).atLeastOnce();
 
     // mock clusters
     expect(m_clusters.getClusters()).andReturn(clusterMap).atLeastOnce();
-    expect(m_clusters.getHostsForCluster(CLUSTER_NAME)).andReturn(hostMap).atLeastOnce();
 
     // mock the definition DAO
     expect(m_definitionDao.findByName(CLUSTER_ID, DEFINITION_NAME)).andReturn(
@@ -152,7 +153,9 @@ public class AgentHeartbeatAlertRunnableTest {
         m_listener.getAlertEventReceivedCount(AlertReceivedEvent.class));
 
     // instantiate and inject mocks
-    AgentHeartbeatAlertRunnable runnable = new AgentHeartbeatAlertRunnable();
+    AgentHeartbeatAlertRunnable runnable = new AgentHeartbeatAlertRunnable(
+        m_definition.getDefinitionName());
+
     m_injector.injectMembers(runnable);
 
     // run the alert
@@ -186,7 +189,9 @@ public class AgentHeartbeatAlertRunnableTest {
         m_listener.getAlertEventReceivedCount(AlertReceivedEvent.class));
 
     // instantiate and inject mocks
-    AgentHeartbeatAlertRunnable runnable = new AgentHeartbeatAlertRunnable();
+    AgentHeartbeatAlertRunnable runnable = new AgentHeartbeatAlertRunnable(
+        m_definition.getDefinitionName());
+
     m_injector.injectMembers(runnable);
 
     // run the alert

ambari-server/src/test/java/org/apache/ambari/server/alerts/AmbariPerformanceRunnableTest.java

@@ -0,0 +1,232 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.alerts;
+
+import static junit.framework.Assert.assertEquals;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.persistence.EntityManager;
+
+import org.apache.ambari.server.actionmanager.ActionManager;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.internal.ClusterResourceProvider;
+import org.apache.ambari.server.events.AlertEvent;
+import org.apache.ambari.server.events.AlertReceivedEvent;
+import org.apache.ambari.server.events.MockEventListener;
+import org.apache.ambari.server.events.publishers.AlertEventPublisher;
+import org.apache.ambari.server.orm.DBAccessor;
+import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
+import org.apache.ambari.server.orm.dao.AlertsDAO;
+import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
+import org.apache.ambari.server.orm.entities.AlertCurrentEntity;
+import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
+import org.apache.ambari.server.state.Alert;
+import org.apache.ambari.server.state.AlertState;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.alert.AlertDefinition;
+import org.apache.ambari.server.state.alert.AlertDefinitionFactory;
+import org.apache.ambari.server.state.alert.ServerSource;
+import org.apache.ambari.server.state.stack.OsFamily;
+import org.easymock.EasyMock;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.eventbus.EventBus;
+import com.google.inject.Binder;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+
+/**
+ * Tests {@link AmbariPerformanceRunnable}.
+ */
+public class AmbariPerformanceRunnableTest {
+
+  private final static long CLUSTER_ID = 1;
+  private final static String CLUSTER_NAME = "c1";
+
+  private final static String DEFINITION_NAME = "ambari_server_performance";
+  private final static String DEFINITION_SERVICE = "AMBARI";
+  private final static String DEFINITION_COMPONENT = "AMBARI_SERVER";
+  private final static String DEFINITION_LABEL = "Mock Definition";
+  private final static int DEFINITION_INTERVAL = 1;
+
+  private Clusters m_clusters;
+  private Cluster m_cluster;
+  private Injector m_injector;
+  private AlertsDAO m_alertsDao;
+  private AlertDefinitionDAO m_definitionDao;
+  private AlertDefinitionEntity m_definition;
+  private List<AlertCurrentEntity> m_currentAlerts = new ArrayList<AlertCurrentEntity>();
+  private MockEventListener m_listener;
+
+  private AlertEventPublisher m_eventPublisher;
+  private EventBus m_synchronizedBus;
+
+  /**
+   *
+   */
+  @Before
+  public void setup() throws Exception {
+    m_injector = Guice.createInjector(new MockModule());
+    m_alertsDao = m_injector.getInstance(AlertsDAO.class);
+    m_definitionDao = m_injector.getInstance(AlertDefinitionDAO.class);
+    m_clusters = m_injector.getInstance(Clusters.class);
+    m_cluster = m_injector.getInstance(Cluster.class);
+    m_eventPublisher = m_injector.getInstance(AlertEventPublisher.class);
+    m_listener = m_injector.getInstance(MockEventListener.class);
+    m_definition = EasyMock.createNiceMock(AlertDefinitionEntity.class);
+
+    // !!! need a synchronous op for testing
+    m_synchronizedBus = new EventBus();
+    Field field = AlertEventPublisher.class.getDeclaredField("m_eventBus");
+    field.setAccessible(true);
+    field.set(m_eventPublisher, m_synchronizedBus);
+
+    // register mock listener
+    m_synchronizedBus.register(m_listener);
+
+    // create the cluster map
+    Map<String,Cluster> clusterMap = new HashMap<String, Cluster>();
+    clusterMap.put(CLUSTER_NAME, m_cluster);
+
+    // mock the definition for the alert
+    expect(m_definition.getDefinitionName()).andReturn(DEFINITION_NAME).atLeastOnce();
+    expect(m_definition.getServiceName()).andReturn(DEFINITION_SERVICE).atLeastOnce();
+    expect(m_definition.getComponentName()).andReturn(DEFINITION_COMPONENT).atLeastOnce();
+    expect(m_definition.getLabel()).andReturn(DEFINITION_LABEL).atLeastOnce();
+    expect(m_definition.getEnabled()).andReturn(true).atLeastOnce();
+    expect(m_definition.getScheduleInterval()).andReturn(DEFINITION_INTERVAL).atLeastOnce();
+
+    // mock the cluster
+    expect(m_cluster.getClusterId()).andReturn(CLUSTER_ID).atLeastOnce();
+
+    // mock clusters
+    expect(m_clusters.getClusters()).andReturn(clusterMap).atLeastOnce();
+
+    // mock the definition DAO
+    expect(m_definitionDao.findByName(CLUSTER_ID, DEFINITION_NAME)).andReturn(
+        m_definition).atLeastOnce();
+
+    // mock the current dao
+    expect(m_alertsDao.findCurrentByCluster(CLUSTER_ID)).andReturn(
+        m_currentAlerts).atLeastOnce();
+
+    // mock the factory and what it returns
+    AlertDefinition definition = new AlertDefinition();
+    definition.setDefinitionId(1L);
+    definition.setName(DEFINITION_NAME);
+    ServerSource source = new ServerSource();
+    definition.setSource(source);
+    AlertDefinitionFactory factory = m_injector.getInstance(AlertDefinitionFactory.class);
+    expect(factory.coerce(EasyMock.anyObject(AlertDefinitionEntity.class))).andReturn(definition).atLeastOnce();
+
+    replay(m_definition, m_cluster, m_clusters, m_definitionDao, m_alertsDao, factory);
+    }
+
+  /**
+   * @throws Exception
+   */
+  @After
+  public void teardown() throws Exception {
+  }
+
+  /**
+   * Tests that the event is triggerd with a status of UNKNOWN.
+   */
+  @Test
+  public void testAlertFiresEvents() {
+    // instantiate and inject mocks
+    AmbariPerformanceRunnable runnable = new AmbariPerformanceRunnable(
+        m_definition.getDefinitionName());
+
+    m_injector.injectMembers(runnable);
+
+    // run the alert
+    runnable.run();
+
+    assertEquals(1,
+        m_listener.getAlertEventReceivedCount(AlertReceivedEvent.class));
+
+    List<AlertEvent> events = m_listener.getAlertEventInstances(AlertReceivedEvent.class);
+    assertEquals(1, events.size());
+
+    AlertReceivedEvent event = (AlertReceivedEvent) events.get(0);
+    Alert alert = event.getAlert();
+    assertEquals("AMBARI", alert.getService());
+    assertEquals("AMBARI_SERVER", alert.getComponent());
+    assertEquals(AlertState.UNKNOWN, alert.getState());
+    assertEquals(DEFINITION_NAME, alert.getName());
+
+    verify(m_cluster, m_clusters, m_definitionDao);
+  }
+
+  /**
+   * Tests that the event is triggerd with a status of UNKNOWN.
+   */
+  @Test
+  public void testThresholdCalculations() {
+    // instantiate and inject mocks
+    AmbariPerformanceRunnable runnable = new AmbariPerformanceRunnable(
+        m_definition.getDefinitionName());
+
+    assertEquals(1, runnable.getThresholdValue(1, 2));
+    assertEquals(1, runnable.getThresholdValue("1", 2));
+    assertEquals(1, runnable.getThresholdValue("1.00", 2));
+    assertEquals(1, runnable.getThresholdValue("foo", 1));
+    assertEquals(1, runnable.getThresholdValue(new Object(), 1));
+  }
+
+  /**
+   *
+   */
+  private class MockModule implements Module {
+    /**
+     *
+     */
+    @Override
+    public void configure(Binder binder) {
+      Cluster cluster = EasyMock.createNiceMock(Cluster.class);
+
+      binder.bind(Clusters.class).toInstance(createNiceMock(Clusters.class));
+      binder.bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class));
+      binder.bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class));
+      binder.bind(Cluster.class).toInstance(cluster);
+      binder.bind(AlertDefinitionDAO.class).toInstance(createNiceMock(AlertDefinitionDAO.class));
+      binder.bind(AlertsDAO.class).toInstance(createNiceMock(AlertsDAO.class));
+      binder.bind(EntityManager.class).toInstance(createNiceMock(EntityManager.class));
+      binder.bind(ActionManager.class).toInstance(createNiceMock(ActionManager.class));
+      binder.bind(HostRoleCommandDAO.class).toInstance(createNiceMock(HostRoleCommandDAO.class));
+      binder.bind(AmbariManagementController.class).toInstance(createNiceMock(AmbariManagementController.class));
+      binder.bind(AlertDefinitionFactory.class).toInstance(createNiceMock(AlertDefinitionFactory.class));
+      binder.bind(ClusterResourceProvider.class).toInstance(createNiceMock(ClusterResourceProvider.class));
+    }
+  }
+}

ambari-server/src/test/java/org/apache/ambari/server/alerts/StaleAlertRunnableTest.java

@@ -151,19 +151,19 @@ public class StaleAlertRunnableTest {
   @Test
   public void testAllAlertsAreCurrent() {
     // create current alerts that are not stale
-    AlertDefinitionEntity definition1 = new AlertDefinitionEntity();
-    definition1.setClusterId(CLUSTER_ID);
-    definition1.setDefinitionName("foo-definition");
-    definition1.setServiceName("HDFS");
-    definition1.setComponentName("NAMENODE");
-    definition1.setEnabled(true);
-    definition1.setScheduleInterval(1);
+    AlertDefinitionEntity definition = new AlertDefinitionEntity();
+    definition.setClusterId(CLUSTER_ID);
+    definition.setDefinitionName("foo-definition");
+    definition.setServiceName("HDFS");
+    definition.setComponentName("NAMENODE");
+    definition.setEnabled(true);
+    definition.setScheduleInterval(1);
 
     AlertCurrentEntity current1 = createNiceMock(AlertCurrentEntity.class);
     AlertHistoryEntity history1 = createNiceMock(AlertHistoryEntity.class);
 
     expect(current1.getAlertHistory()).andReturn(history1).atLeastOnce();
-    expect(history1.getAlertDefinition()).andReturn(definition1).atLeastOnce();
+    expect(history1.getAlertDefinition()).andReturn(definition).atLeastOnce();
 
     expect(current1.getMaintenanceState()).andReturn(MaintenanceState.OFF).atLeastOnce();
     expect(current1.getLatestTimestamp()).andReturn(System.currentTimeMillis()).atLeastOnce();
@@ -177,7 +177,7 @@ public class StaleAlertRunnableTest {
         m_listener.getAlertEventReceivedCount(AlertReceivedEvent.class));
 
     // instantiate and inject mocks
-    StaleAlertRunnable runnable = new StaleAlertRunnable();
+    StaleAlertRunnable runnable = new StaleAlertRunnable(m_definition.getDefinitionName());
     m_injector.injectMembers(runnable);
 
     // run the alert
@@ -205,20 +205,20 @@ public class StaleAlertRunnableTest {
   @Test
   public void testStaleAlert() {
     // create current alerts that are not stale
-    AlertDefinitionEntity definition1 = new AlertDefinitionEntity();
-    definition1.setClusterId(CLUSTER_ID);
-    definition1.setDefinitionName("foo-definition");
-    definition1.setServiceName("HDFS");
-    definition1.setComponentName("NAMENODE");
-    definition1.setEnabled(true);
-    definition1.setScheduleInterval(1);
+    AlertDefinitionEntity definition = new AlertDefinitionEntity();
+    definition.setClusterId(CLUSTER_ID);
+    definition.setDefinitionName("foo-definition");
+    definition.setServiceName("HDFS");
+    definition.setComponentName("NAMENODE");
+    definition.setEnabled(true);
+    definition.setScheduleInterval(1);
 
     // create current alerts that are stale
     AlertCurrentEntity current1 = createNiceMock(AlertCurrentEntity.class);
     AlertHistoryEntity history1 = createNiceMock(AlertHistoryEntity.class);
 
     expect(current1.getAlertHistory()).andReturn(history1).atLeastOnce();
-    expect(history1.getAlertDefinition()).andReturn(definition1).atLeastOnce();
+    expect(history1.getAlertDefinition()).andReturn(definition).atLeastOnce();
 
     // a really old timestampt to trigger the alert
     expect(current1.getMaintenanceState()).andReturn(MaintenanceState.OFF).atLeastOnce();
@@ -233,7 +233,7 @@ public class StaleAlertRunnableTest {
         m_listener.getAlertEventReceivedCount(AlertReceivedEvent.class));
 
     // instantiate and inject mocks
-    StaleAlertRunnable runnable = new StaleAlertRunnable();
+    StaleAlertRunnable runnable = new StaleAlertRunnable(m_definition.getDefinitionName());
     m_injector.injectMembers(runnable);
 
     // run the alert
@@ -261,13 +261,13 @@ public class StaleAlertRunnableTest {
   @Test
   public void testStaleAlertInMaintenaceMode() {
     // create current alerts that are not stale
-    AlertDefinitionEntity definition1 = new AlertDefinitionEntity();
-    definition1.setClusterId(CLUSTER_ID);
-    definition1.setDefinitionName("foo-definition");
-    definition1.setServiceName("HDFS");
-    definition1.setComponentName("NAMENODE");
-    definition1.setEnabled(true);
-    definition1.setScheduleInterval(1);
+    AlertDefinitionEntity definition = new AlertDefinitionEntity();
+    definition.setClusterId(CLUSTER_ID);
+    definition.setDefinitionName("foo-definition");
+    definition.setServiceName("HDFS");
+    definition.setComponentName("NAMENODE");
+    definition.setEnabled(true);
+    definition.setScheduleInterval(1);
 
     // create current alerts where 1 is stale but in maintence mode
     AlertCurrentEntity current1 = createNiceMock(AlertCurrentEntity.class);
@@ -276,10 +276,10 @@ public class StaleAlertRunnableTest {
     AlertHistoryEntity history2 = createNiceMock(AlertHistoryEntity.class);
 
     expect(current1.getAlertHistory()).andReturn(history1).atLeastOnce();
-    expect(history1.getAlertDefinition()).andReturn(definition1).atLeastOnce();
+    expect(history1.getAlertDefinition()).andReturn(definition).atLeastOnce();
 
     expect(current2.getAlertHistory()).andReturn(history2).atLeastOnce();
-    expect(history2.getAlertDefinition()).andReturn(definition1).atLeastOnce();
+    expect(history2.getAlertDefinition()).andReturn(definition).atLeastOnce();
 
     // maintenance mode with a really old timestamp
     expect(current1.getMaintenanceState()).andReturn(MaintenanceState.ON).atLeastOnce();
@@ -299,7 +299,7 @@ public class StaleAlertRunnableTest {
         m_listener.getAlertEventReceivedCount(AlertReceivedEvent.class));
 
     // instantiate and inject mocks
-    StaleAlertRunnable runnable = new StaleAlertRunnable();
+    StaleAlertRunnable runnable = new StaleAlertRunnable(m_definition.getDefinitionName());
     m_injector.injectMembers(runnable);
 
     // run the alert

ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java

@@ -18,12 +18,33 @@
 
 package org.apache.ambari.server.api.services;
 
-import com.google.gson.Gson;
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.util.Modules;
-import junit.framework.Assert;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.FileReader;
+import java.lang.reflect.Field;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.UUID;
+
+import javax.persistence.EntityManager;
+
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.StackAccessException;
 import org.apache.ambari.server.configuration.Configuration;
@@ -77,31 +98,13 @@ import org.junit.rules.TemporaryFolder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.persistence.EntityManager;
-import java.io.File;
-import java.io.FileReader;
-import java.lang.reflect.Field;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.UUID;
+import com.google.gson.Gson;
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.util.Modules;
 
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.reset;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import junit.framework.Assert;
 
 public class AmbariMetaInfoTest {
 
@@ -1858,7 +1861,7 @@ public class AmbariMetaInfoTest {
 
     AlertDefinitionDAO dao = injector.getInstance(AlertDefinitionDAO.class);
     List<AlertDefinitionEntity> definitions = dao.findAll(clusterId);
-    assertEquals(9, definitions.size());
+    assertEquals(10, definitions.size());
 
     // figure out how many of these alerts were merged into from the
     // non-stack alerts.json
@@ -1871,7 +1874,7 @@ public class AmbariMetaInfoTest {
     }
 
     assertEquals(1, hostAlertCount);
-    assertEquals(8, definitions.size() - hostAlertCount);
+    assertEquals(9, definitions.size() - hostAlertCount);
 
     for (AlertDefinitionEntity definition : definitions) {
       definition.setScheduleInterval(28);
@@ -1881,7 +1884,7 @@ public class AmbariMetaInfoTest {
     metaInfo.reconcileAlertDefinitions(clusters);
 
     definitions = dao.findAll();
-    assertEquals(9, definitions.size());
+    assertEquals(10, definitions.size());
 
     for (AlertDefinitionEntity definition : definitions) {
       assertEquals(28, definition.getScheduleInterval().intValue());
@@ -1890,7 +1893,7 @@ public class AmbariMetaInfoTest {
     // find all enabled for the cluster should find 6 (the ones from HDFS;
     // it will not find the agent alert since it's not bound to the cluster)
     definitions = dao.findAllEnabled(cluster.getClusterId());
-    assertEquals(8, definitions.size());
+    assertEquals(9, definitions.size());
 
     // create new definition
     AlertDefinitionEntity entity = new AlertDefinitionEntity();
@@ -1909,19 +1912,19 @@ public class AmbariMetaInfoTest {
 
     // verify the new definition is found (6 HDFS + 1 new one)
     definitions = dao.findAllEnabled(cluster.getClusterId());
-    assertEquals(9, definitions.size());
+    assertEquals(10, definitions.size());
 
     // reconcile, which should disable our bad definition
     metaInfo.reconcileAlertDefinitions(clusters);
 
     // find all enabled for the cluster should find 6
     definitions = dao.findAllEnabled(cluster.getClusterId());
-    assertEquals(8, definitions.size());
+    assertEquals(9, definitions.size());
 
     // find all should find 6 HDFS + 1 disabled + 1 agent alert + 2 server
     // alerts
     definitions = dao.findAll();
-    assertEquals(10, definitions.size());
+    assertEquals(11, definitions.size());
 
     entity = dao.findById(entity.getDefinitionId());
     assertFalse(entity.getEnabled());

ambari-server/src/test/java/org/apache/ambari/server/metadata/AgentAlertDefinitionsTest.java

@@ -19,8 +19,6 @@ package org.apache.ambari.server.metadata;
 
 import java.util.List;
 
-import junit.framework.Assert;
-
 import org.apache.ambari.server.controller.RootServiceResponseFactory.Components;
 import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
 import org.apache.ambari.server.state.alert.AlertDefinition;
@@ -30,6 +28,8 @@ import org.junit.Test;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
 
+import junit.framework.Assert;
+
 /**
  * Tets {@link AmbariServiceAlertDefinitions}.
  */
@@ -66,7 +66,7 @@ public class AgentAlertDefinitionsTest {
   public void testLoadingServertAlerts() {
     AmbariServiceAlertDefinitions ambariServiceAlertDefinitions = m_injector.getInstance(AmbariServiceAlertDefinitions.class);
     List<AlertDefinition> definitions = ambariServiceAlertDefinitions.getServerDefinitions();
-    Assert.assertEquals(2, definitions.size());
+    Assert.assertEquals(3, definitions.size());
 
     for (AlertDefinition definition : definitions) {
       Assert.assertEquals(Components.AMBARI_SERVER.name(),