AMBARI-10348 - Alerts: Generate Server Side Alerts For Agent Health and Alert Staleness (jonathanhurley)

ambari-server/src/main/java/org/apache/ambari/server/alerts/AgentHeartbeatAlertRunnable.java

@@ -0,0 +1,185 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.alerts;
+
+import java.text.MessageFormat;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.ambari.server.events.AlertEvent;
+import org.apache.ambari.server.events.AlertReceivedEvent;
+import org.apache.ambari.server.events.publishers.AlertEventPublisher;
+import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
+import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
+import org.apache.ambari.server.state.Alert;
+import org.apache.ambari.server.state.AlertState;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Host;
+import org.apache.ambari.server.state.HostState;
+import org.apache.ambari.server.state.services.AmbariServerAlertService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+
+/**
+ * The {@link AgentHeartbeatAlertRunnable} is used by the
+ * {@link AmbariServerAlertService} to check agent heartbeats and fire alert
+ * events when agents are not reachable.
+ */
+public class AgentHeartbeatAlertRunnable implements Runnable {
+
+  /**
+   * Logger.
+   */
+  private final static Logger LOG = LoggerFactory.getLogger(AgentHeartbeatAlertRunnable.class);
+
+  /**
+   * The unique name for the alert definition that governs this service.
+   */
+  private static final String HEARTBEAT_DEFINITION_NAME = "ambari_server_agent_heartbeat";
+
+  /**
+   * Agent initializing message.
+   */
+  private static final String INIT_MSG = "{0} is initializing";
+
+  /**
+   * Agent healthy message.
+   */
+  private static final String HEALTHY_MSG = "{0} is healthy";
+
+  /**
+   * Agent waiting for status updates message.
+   */
+  private static final String STATUS_UPDATE_MSG = "{0} is waiting for status updates";
+
+  /**
+   * Agent is not heartbeating message.
+   */
+  private static final String HEARTBEAT_LOST_MSG = "{0} is not sending heartbeats";
+
+  /**
+   * Agent is not healthy message.
+   */
+  private static final String UNHEALTHY_MSG = "{0} is not healthy";
+
+  /**
+   * Unknown agent state message.
+   */
+  private static final String UNKNOWN_MSG = "{0} has an unknown state of {1}";
+
+  /**
+   * Used for looking up alert definitions.
+   */
+  @Inject
+  private AlertDefinitionDAO m_dao;
+
+  /**
+   * Used to get alert definitions to use when generating alert instances.
+   */
+  @Inject
+  private Provider<Clusters> m_clustersProvider;
+
+  /**
+   * Publishes {@link AlertEvent} instances.
+   */
+  @Inject
+  private AlertEventPublisher m_alertEventPublisher;
+
+  /**
+   * Constructor. Required for type introspection by
+   * {@link AmbariServerAlertService}.
+   */
+  public AgentHeartbeatAlertRunnable() {
+  }
+
+  @Override
+  public void run() {
+    try {
+      Map<String, Cluster> clusterMap = m_clustersProvider.get().getClusters();
+      for (Cluster cluster : clusterMap.values()) {
+        AlertDefinitionEntity entity = m_dao.findByName(cluster.getClusterId(),
+            HEARTBEAT_DEFINITION_NAME);
+
+        // skip this cluster if the runnable's alert definition is missing or
+        // disabled
+        if (null == entity || !entity.getEnabled()) {
+          continue;
+        }
+
+        long alertTimestamp = System.currentTimeMillis();
+
+        Map<String, Host> hostMap = m_clustersProvider.get().getHostsForCluster(
+            cluster.getClusterName());
+
+        Set<Entry<String, Host>> entries = hostMap.entrySet();
+        for (Entry<String, Host> entry : entries) {
+          String hostName = entry.getKey();
+          Host host = entry.getValue();
+
+          String alertText;
+          AlertState alertState = AlertState.OK;
+          HostState hostState = host.getState();
+
+          switch (hostState) {
+            case INIT:
+              alertText = MessageFormat.format(INIT_MSG, hostName);
+              break;
+            case HEALTHY:
+              alertText = MessageFormat.format(HEALTHY_MSG, hostName);
+              break;
+            case WAITING_FOR_HOST_STATUS_UPDATES:
+              alertText = MessageFormat.format(STATUS_UPDATE_MSG, hostName);
+              break;
+            case HEARTBEAT_LOST:
+              alertState = AlertState.CRITICAL;
+              alertText = MessageFormat.format(HEARTBEAT_LOST_MSG, hostName);
+              break;
+            case UNHEALTHY:
+              alertState = AlertState.CRITICAL;
+              alertText = MessageFormat.format(UNHEALTHY_MSG, hostName);
+            default:
+              alertState = AlertState.UNKNOWN;
+              alertText = MessageFormat.format(UNKNOWN_MSG, hostName, hostState);
+              break;
+          }
+
+          Alert alert = new Alert(entity.getDefinitionName(), null,
+              entity.getServiceName(), entity.getComponentName(), hostName,
+              alertState);
+
+          alert.setLabel(entity.getLabel());
+          alert.setText(alertText);
+          alert.setTimestamp(alertTimestamp);
+
+          AlertReceivedEvent event = new AlertReceivedEvent(
+              cluster.getClusterId(), alert);
+
+          m_alertEventPublisher.publish(event);
+        }
+      }
+    } catch (Exception exception) {
+      LOG.error("Unable to run the {} alert", HEARTBEAT_DEFINITION_NAME,
+          exception);
+    }
+  }
+}

ambari-server/src/main/java/org/apache/ambari/server/alerts/StaleAlertRunnable.java

@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.alerts;
+
+import java.text.MessageFormat;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.ambari.server.events.AlertEvent;
+import org.apache.ambari.server.events.AlertReceivedEvent;
+import org.apache.ambari.server.events.publishers.AlertEventPublisher;
+import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
+import org.apache.ambari.server.orm.dao.AlertsDAO;
+import org.apache.ambari.server.orm.entities.AlertCurrentEntity;
+import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
+import org.apache.ambari.server.orm.entities.AlertHistoryEntity;
+import org.apache.ambari.server.state.Alert;
+import org.apache.ambari.server.state.AlertState;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.MaintenanceState;
+import org.apache.ambari.server.state.alert.SourceType;
+import org.apache.ambari.server.state.services.AmbariServerAlertService;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+
+/**
+ * The {@link StaleAlertRunnable} is used by the
+ * {@link AmbariServerAlertService} to check agent heartbeats and fire alert
+ * events when an agent host changes state.
+ */
+public class StaleAlertRunnable implements Runnable {
+
+  /**
+   * Logger.
+   */
+  private final static Logger LOG = LoggerFactory.getLogger(StaleAlertRunnable.class);
+
+  /**
+   * The unique name for the alert definition that governs this service.
+   */
+  private static final String STALE_ALERT_DEFINITION_NAME = "ambari_server_stale_alerts";
+
+  /**
+   * The message for the alert when all services have run in their designated
+   * intervals.
+   */
+  private static final String ALL_ALERTS_CURRENT_MSG = "All alerts have run within their time intervals.";
+
+  /**
+   * The message to use when alerts are detected as stale.
+   */
+  private static final String STALE_ALERTS_MSG = "There are {0} stale alerts from {1} host(s): {2}";
+
+  /**
+   * Convert the minutes for the delay of an alert into milliseconds.
+   */
+  private static final long MINUTE_TO_MS_CONVERSION = 60L * 1000L;
+
+  /**
+   * Used to get the current alerts and the last time they ran.
+   */
+  @Inject
+  private AlertsDAO m_alertsDao;
+
+  /**
+   * Used to get alert definitions to use when generating alert instances.
+   */
+  @Inject
+  private Provider<Clusters> m_clustersProvider;
+
+  /**
+   * Used for looking up alert definitions.
+   */
+  @Inject
+  private AlertDefinitionDAO m_dao;
+
+  /**
+   * Publishes {@link AlertEvent} instances.
+   */
+  @Inject
+  private AlertEventPublisher m_alertEventPublisher;
+
+  /**
+   * Constructor. Required for type introspection by
+   * {@link AmbariServerAlertService}.
+   */
+  public StaleAlertRunnable() {
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void run() {
+    try {
+      Map<String, Cluster> clusterMap = m_clustersProvider.get().getClusters();
+      for (Cluster cluster : clusterMap.values()) {
+        AlertDefinitionEntity entity = m_dao.findByName(cluster.getClusterId(),
+            STALE_ALERT_DEFINITION_NAME);
+
+        // skip this cluster if the runnable's alert definition is missing or
+        // disabled
+        if (null == entity || !entity.getEnabled()) {
+          continue;
+        }
+
+        long now = System.currentTimeMillis();
+        Set<String> staleAlerts = new HashSet<String>();
+        Set<String> hostsWithStaleAlerts = new HashSet<String>();
+
+        // get the cluster's current alerts
+        List<AlertCurrentEntity> currentAlerts = m_alertsDao.findCurrentByCluster(cluster.getClusterId());
+
+        // for each current alert, check to see if the last time it ran is
+        // more than 2x its interval value (indicating it hasn't run)
+        for (AlertCurrentEntity current : currentAlerts) {
+          AlertHistoryEntity history = current.getAlertHistory();
+          AlertDefinitionEntity definition = history.getAlertDefinition();
+
+          // skip aggregates as they are special
+          if (definition.getSourceType() == SourceType.AGGREGATE) {
+            continue;
+          }
+
+          // skip alerts in maintenance mode
+          if (current.getMaintenanceState() != MaintenanceState.OFF) {
+            continue;
+          }
+
+          // skip alerts that have not run yet
+          if (current.getLatestTimestamp() == 0) {
+            continue;
+          }
+
+          // convert minutes to milliseconds for the definition's interval
+          long intervalInMillis = definition.getScheduleInterval()
+              * MINUTE_TO_MS_CONVERSION;
+
+          // if the last time it was run is >= 2x the interval, it's stale
+          long timeDifference = now - current.getLatestTimestamp();
+          if (timeDifference >= 2 * intervalInMillis) {
+            // keep track of the definition
+            staleAlerts.add(definition.getLabel());
+
+            // keek track of the host, if not null
+            if (null != history.getHostName()) {
+              hostsWithStaleAlerts.add( history.getHostName() );
+            }
+          }
+        }
+
+        AlertState alertState = AlertState.OK;
+        String alertText = ALL_ALERTS_CURRENT_MSG;
+
+        // if there are stale alerts, mark as CRITICAL with the list of
+        // alerts
+        if( !staleAlerts.isEmpty() ){
+          alertState = AlertState.CRITICAL;
+          alertText = MessageFormat.format(STALE_ALERTS_MSG,
+              staleAlerts.size(), hostsWithStaleAlerts.size(),
+              StringUtils.join(staleAlerts, ", "));
+        }
+
+        Alert alert = new Alert(entity.getDefinitionName(), null,
+            entity.getServiceName(), entity.getComponentName(), null,
+            alertState);
+
+        alert.setLabel(entity.getLabel());
+        alert.setText(alertText);
+        alert.setTimestamp(now);
+
+        AlertReceivedEvent event = new AlertReceivedEvent(
+            cluster.getClusterId(), alert);
+
+        m_alertEventPublisher.publish(event);
+      }
+    } catch (Exception exception) {
+      LOG.error("Unable to run the {} alert", STALE_ALERT_DEFINITION_NAME,
+          exception);
+    }
+  }
+}

ambari-server/src/main/java/org/apache/ambari/server/api/services/AlertDefinitionService.java

@@ -41,20 +41,19 @@ import org.apache.ambari.server.controller.spi.Resource;
 public class AlertDefinitionService extends BaseService {
 
   private String clusterName = null;
-  
+
   AlertDefinitionService(String clusterName) {
     this.clusterName = clusterName;
   }
-  
+
   @GET
   @Produces("text/plain")
-  public Response getDefinitions(String body,
-      @Context HttpHeaders headers,
+  public Response getDefinitions(@Context HttpHeaders headers,
       @Context UriInfo ui) {
-    return handleRequest(headers, body, ui, Request.Type.GET,
+    return handleRequest(headers, null, ui, Request.Type.GET,
       createResourceInstance(clusterName, null));
   }
-  
+
   @POST
   @Produces("text/plain")
   public Response createDefinition(String body,
@@ -63,7 +62,7 @@ public class AlertDefinitionService extends BaseService {
     return handleRequest(headers, body, ui, Request.Type.POST,
       createResourceInstance(clusterName, null));
   }
-  
+
   @PUT
   @Path("{alertDefinitionId}")
   @Produces("text/plain")
@@ -74,7 +73,7 @@ public class AlertDefinitionService extends BaseService {
     return handleRequest(headers, body, ui, Request.Type.PUT,
       createResourceInstance(clusterName, id));
   }
-  
+
   @DELETE
   @Path("{alertDefinitionId}")
   @Produces("text/plain")
@@ -86,19 +85,18 @@ public class AlertDefinitionService extends BaseService {
       createResourceInstance(clusterName, id));
   }
 
-  
+
   @GET
   @Path("{alertDefinitionId}")
   @Produces("text/plain")
-  public Response getDefinitions(String body,
-      @Context HttpHeaders headers,
+  public Response getDefinitions(@Context HttpHeaders headers,
       @Context UriInfo ui,
       @PathParam("alertDefinitionId") Long id) {
-    return handleRequest(headers, body, ui, Request.Type.GET,
+    return handleRequest(headers, null, ui, Request.Type.GET,
       createResourceInstance(clusterName, id));
   }
-  
-  
+
+
   /**
    * Create a request schedule resource instance
    * @param clusterName
@@ -113,5 +111,5 @@ public class AlertDefinitionService extends BaseService {
 
     return createResource(Resource.Type.AlertDefinition, mapIds);
   }
-  
+
 }

ambari-server/src/main/java/org/apache/ambari/server/api/services/AlertGroupService.java

@@ -57,18 +57,18 @@ public class AlertGroupService extends BaseService {
 
   @GET
   @Produces("text/plain")
-  public Response getGroups(String body, @Context HttpHeaders headers,
+  public Response getGroups(@Context HttpHeaders headers,
       @Context UriInfo ui) {
-    return handleRequest(headers, body, ui, Request.Type.GET,
+    return handleRequest(headers, null, ui, Request.Type.GET,
         createAlertGroupResource(m_clusterName, null));
   }
 
   @GET
   @Produces("text/plain")
   @Path("{groupId}")
-  public Response getGroup(String body, @Context HttpHeaders headers,
+  public Response getGroup(@Context HttpHeaders headers,
       @Context UriInfo ui, @PathParam("groupId") Long groupId) {
-    return handleRequest(headers, body, ui, Request.Type.GET,
+    return handleRequest(headers, null, ui, Request.Type.GET,
         createAlertGroupResource(m_clusterName, groupId));
   }
 

ambari-server/src/main/java/org/apache/ambari/server/api/services/AlertHistoryService.java

@@ -60,20 +60,19 @@ public class AlertHistoryService extends BaseService {
 
   @GET
   @Produces("text/plain")
-  public Response getHistories(String body,
-      @Context HttpHeaders headers,
+  public Response getHistories(@Context HttpHeaders headers,
       @Context UriInfo ui) {
-    return handleRequest(headers, body, ui, Request.Type.GET,
+    return handleRequest(headers, null, ui, Request.Type.GET,
         createResourceInstance(clusterName, null));
   }
 
   @GET
   @Path("{alertHistoryId}")
   @Produces("text/plain")
-  public Response getHistory(String body,
+  public Response getHistory(
       @Context HttpHeaders headers,
       @Context UriInfo ui, @PathParam("alertHistoryId") Long id) {
-    return handleRequest(headers, body, ui, Request.Type.GET,
+    return handleRequest(headers, null, ui, Request.Type.GET,
         createResourceInstance(clusterName, id));
   }
 

ambari-server/src/main/java/org/apache/ambari/server/api/services/AlertNoticeService.java

@@ -51,20 +51,20 @@ public class AlertNoticeService extends BaseService {
 
   @GET
   @Produces("text/plain")
-  public Response getNotices(String body,
+  public Response getNotices(
       @Context HttpHeaders headers,
       @Context UriInfo ui) {
-    return handleRequest(headers, body, ui, Request.Type.GET,
+    return handleRequest(headers, null, ui, Request.Type.GET,
         createResourceInstance(clusterName, null));
   }
 
   @GET
   @Path("{alertNoticeId}")
   @Produces("text/plain")
-  public Response getNotice(String body,
+  public Response getNotice(
       @Context HttpHeaders headers,
       @Context UriInfo ui, @PathParam("alertNoticeId") Long id) {
-    return handleRequest(headers, body, ui, Request.Type.GET,
+    return handleRequest(headers, null, ui, Request.Type.GET,
         createResourceInstance(clusterName, id));
   }
 

ambari-server/src/main/java/org/apache/ambari/server/api/services/AlertService.java

@@ -40,7 +40,7 @@ public class AlertService extends BaseService {
   private String clusterName = null;
   private String serviceName = null;
   private String hostName = null;
-  
+
   AlertService(String clusterName, String serviceName, String hostName) {
     this.clusterName = clusterName;
     this.serviceName = serviceName;
@@ -52,13 +52,13 @@ public class AlertService extends BaseService {
    */
   @GET
   @Produces("text/plain")
-  public Response getAlerts(String body,
+  public Response getAlerts(
       @Context HttpHeaders headers,
       @Context UriInfo ui) {
-    return handleRequest(headers, body, ui, Request.Type.GET,
+    return handleRequest(headers, null, ui, Request.Type.GET,
       createResourceInstance(null));
   }
-  
+
 
   /**
    * Gets a specific alert's instance
@@ -66,15 +66,15 @@ public class AlertService extends BaseService {
   @GET
   @Path("{alertId}")
   @Produces("text/plain")
-  public Response getAlert(String body,
+  public Response getAlert(
       @Context HttpHeaders headers,
       @Context UriInfo ui,
       @PathParam("alertId") Long id) {
-    return handleRequest(headers, body, ui, Request.Type.GET,
+    return handleRequest(headers, null, ui, Request.Type.GET,
       createResourceInstance(id));
   }
-  
-  
+
+
   /**
    * Create an alert resource instance
    * @param alertId the alert id, if requesting a specific one
@@ -89,5 +89,5 @@ public class AlertService extends BaseService {
 
     return createResource(Resource.Type.Alert, mapIds);
   }
-  
+
 }

ambari-server/src/main/java/org/apache/ambari/server/api/services/AlertTargetService.java

@@ -45,18 +45,18 @@ public class AlertTargetService extends BaseService {
 
   @GET
   @Produces("text/plain")
-  public Response getTargets(String body, @Context HttpHeaders headers,
+  public Response getTargets(@Context HttpHeaders headers,
       @Context UriInfo ui) {
-    return handleRequest(headers, body, ui, Request.Type.GET,
+    return handleRequest(headers, null, ui, Request.Type.GET,
         createAlertTargetResource(null));
   }
 
   @GET
   @Produces("text/plain")
   @Path("{targetId}")
-  public Response getTargets(String body, @Context HttpHeaders headers,
+  public Response getTargets(@Context HttpHeaders headers,
       @Context UriInfo ui, @PathParam("targetId") Long targetId) {
-    return handleRequest(headers, body, ui, Request.Type.GET,
+    return handleRequest(headers, null, ui, Request.Type.GET,
         createAlertTargetResource(targetId));
   }
 

ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java

@@ -40,14 +40,13 @@ import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.ParentObjectNotFoundException;
 import org.apache.ambari.server.StackAccessException;
 import org.apache.ambari.server.configuration.Configuration;
-import org.apache.ambari.server.controller.RootServiceResponseFactory.Components;
 import org.apache.ambari.server.controller.RootServiceResponseFactory.Services;
 import org.apache.ambari.server.customactions.ActionDefinition;
 import org.apache.ambari.server.customactions.ActionDefinitionManager;
 import org.apache.ambari.server.events.AlertDefinitionDisabledEvent;
 import org.apache.ambari.server.events.AlertDefinitionRegistrationEvent;
 import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
-import org.apache.ambari.server.metadata.AgentAlertDefinitions;
+import org.apache.ambari.server.metadata.AmbariServiceAlertDefinitions;
 import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
 import org.apache.ambari.server.orm.dao.MetainfoDAO;
 import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
@@ -58,10 +57,8 @@ import org.apache.ambari.server.stack.StackManagerFactory;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.ComponentInfo;
-import org.apache.ambari.server.state.ConfigHelper;
 import org.apache.ambari.server.state.DependencyInfo;
 import org.apache.ambari.server.state.OperatingSystemInfo;
-import org.apache.ambari.server.state.PropertyDependencyInfo;
 import org.apache.ambari.server.state.PropertyInfo;
 import org.apache.ambari.server.state.RepositoryInfo;
 import org.apache.ambari.server.state.Service;
@@ -170,7 +167,7 @@ public class AmbariMetaInfo {
    * All of the {@link AlertDefinition}s that are scoped for the agents.
    */
   @Inject
-  private AgentAlertDefinitions agentAlertDefinitions;
+  private AmbariServiceAlertDefinitions ambariServiceAlertDefinitions;
 
   /**
    * Publishes the following events:
@@ -1039,8 +1036,8 @@ public class AmbariMetaInfo {
         }
       }
 
-      // host-only alert definitions
-      List<AlertDefinition> agentDefinitions = agentAlertDefinitions.getDefinitions();
+      // ambari agent host-only alert definitions
+      List<AlertDefinition> agentDefinitions = ambariServiceAlertDefinitions.getAgentDefinitions();
       for (AlertDefinition agentDefinition : agentDefinitions) {
         AlertDefinitionEntity entity = mappedEntities.get(agentDefinition.getName());
 
@@ -1051,6 +1048,18 @@ public class AmbariMetaInfo {
         }
       }
 
+      // ambari server host-only alert definitions
+      List<AlertDefinition> serverDefinitions = ambariServiceAlertDefinitions.getServerDefinitions();
+      for (AlertDefinition serverDefinition : serverDefinitions) {
+        AlertDefinitionEntity entity = mappedEntities.get(serverDefinition.getName());
+
+        // no entity means this is new; create a new entity
+        if (null == entity) {
+          entity = alertDefinitionFactory.coerce(clusterId, serverDefinition);
+          persist.add(entity);
+        }
+      }
+
       // persist any new or updated definition
       for (AlertDefinitionEntity entity : persist) {
         if (LOG.isDebugEnabled()) {
@@ -1083,8 +1092,7 @@ public class AmbariMetaInfo {
         String componentName = definition.getComponentName();
 
         // the AMBARI service is special, skip it here
-        if (Services.AMBARI.name().equals(serviceName)
-            && Components.AMBARI_AGENT.name().equals(componentName)) {
+        if (Services.AMBARI.name().equals(serviceName)) {
           continue;
         }
 

ambari-server/src/main/java/org/apache/ambari/server/events/listeners/alerts/AlertHostListener.java

@@ -17,6 +17,7 @@
  */
 package org.apache.ambari.server.events.listeners.alerts;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.locks.Lock;
@@ -28,7 +29,7 @@ import org.apache.ambari.server.events.AlertHashInvalidationEvent;
 import org.apache.ambari.server.events.HostAddedEvent;
 import org.apache.ambari.server.events.HostRemovedEvent;
 import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
-import org.apache.ambari.server.metadata.AgentAlertDefinitions;
+import org.apache.ambari.server.metadata.AmbariServiceAlertDefinitions;
 import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
 import org.apache.ambari.server.orm.dao.AlertsDAO;
 import org.apache.ambari.server.orm.entities.AlertCurrentEntity;
@@ -78,7 +79,7 @@ public class AlertHostListener {
    * All of the {@link AlertDefinition}s that are scoped for the agents.
    */
   @Inject
-  private AgentAlertDefinitions m_agentAlertDefinitions;
+  private AmbariServiceAlertDefinitions m_ambariServiceAlertDefinitions;
 
   /**
    * Used when a host is added to a cluster to coerce an {@link AlertDefinition}
@@ -121,14 +122,19 @@ public class AlertHostListener {
     long clusterId = event.getClusterId();
 
     // load the host-only alert definitions
-    List<AlertDefinition> agentDefinitions = m_agentAlertDefinitions.getDefinitions();
+    List<AlertDefinition> agentDefinitions = m_ambariServiceAlertDefinitions.getAgentDefinitions();
+    List<AlertDefinition> serverDefinitions = m_ambariServiceAlertDefinitions.getServerDefinitions();
+
+    List<AlertDefinition> ambariServiceDefinitions = new ArrayList<AlertDefinition>();
+    ambariServiceDefinitions.addAll(agentDefinitions);
+    ambariServiceDefinitions.addAll(serverDefinitions);
 
     // lock to prevent multiple threads from trying to create alert
     // definitions at the same time
     m_hostAlertLock.lock();
 
     try {
-      for (AlertDefinition agentDefinition : agentDefinitions) {
+      for (AlertDefinition agentDefinition : ambariServiceDefinitions) {
         AlertDefinitionEntity definition = m_alertDefinitionDao.findByName(
             clusterId, agentDefinition.getName());
 
@@ -141,7 +147,7 @@ public class AlertHostListener {
             m_alertDefinitionDao.create(definition);
           } catch (AmbariException ambariException) {
             LOG.error(
-                "Unable to create a host alert definition name {} in cluster {}",
+                "Unable to create an alert definition named {} in cluster {}",
                 definition.getDefinitionName(), definition.getClusterId(),
                 ambariException);
           }

ambari-server/src/main/java/org/apache/ambari/server/events/listeners/alerts/AlertMaintenanceModeListener.java

@@ -22,7 +22,6 @@ import java.util.List;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.EagerSingleton;
 import org.apache.ambari.server.controller.MaintenanceStateHelper;
-import org.apache.ambari.server.controller.RootServiceResponseFactory.Components;
 import org.apache.ambari.server.controller.RootServiceResponseFactory.Services;
 import org.apache.ambari.server.events.MaintenanceModeEvent;
 import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
@@ -112,8 +111,7 @@ public class AlertMaintenanceModeListener {
       try {
         // although AMBARI is a service, it's not really a service and would
         // fail in this loop; so handle it specifically
-        if (Services.AMBARI.name().equals(serviceName)
-            && Components.AMBARI_AGENT.name().equals(componentName)) {
+        if (Services.AMBARI.name().equals(serviceName)) {
 
           // if this alert is an AMBARI_AGENT alert, then the only maintenance
           // state that affects it is a host maintenance state

ambari-server/src/main/java/org/apache/ambari/server/metadata/AgentAlertDefinitions.java

@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.server.metadata;
-
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.ambari.server.controller.RootServiceResponseFactory.Components;
-import org.apache.ambari.server.state.alert.AlertDefinition;
-import org.apache.ambari.server.state.alert.AlertDefinitionFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-/**
- * The {@link AgentAlertDefinitions} class is used to represent the alerts
- * defined in {@code alerts.json} which are for {@link Components#AMBARI_AGENT}.
- * These alerts are bound to the host and are not part of a cluster or hadoop
- * service.
- */
-@Singleton
-public class AgentAlertDefinitions {
-
-  /**
-   * Logger.
-   */
-  private final static Logger LOG = LoggerFactory.getLogger(AgentAlertDefinitions.class);
-
-  /**
-   * The agent host definitions.
-   */
-  private List<AlertDefinition> m_definitions = null;
-
-  /**
-   * The factory that will load the definitions from the alerts.json file.
-   */
-  @Inject
-  private AlertDefinitionFactory m_factory;
-
-  /**
-   * Gets all of the {@link AlertDefinition}s that exist on the path for all
-   * agent hosts.
-   *
-   * @return the alerts with {@link Components#AMBARI_AGENT} as the component
-   *         and {@code AMBARI} as the service.
-   */
-  public List<AlertDefinition> getDefinitions() {
-    if (null == m_definitions) {
-      m_definitions = new ArrayList<AlertDefinition>();
-
-      InputStream inputStream = ClassLoader.getSystemResourceAsStream("alerts.json");
-      InputStreamReader reader = new InputStreamReader(inputStream);
-
-      try {
-        Set<AlertDefinition> definitions = m_factory.getAlertDefinitions(
-            reader, "AMBARI");
-
-        String agentComponent = Components.AMBARI_AGENT.name();
-
-        for (AlertDefinition definition : definitions) {
-          if (agentComponent.equals(definition.getComponentName())) {
-            m_definitions.add(definition);
-          }
-        }
-
-      } catch (Exception exception) {
-        LOG.error("Unable to load the Ambari alerts JSON file", exception);
-      }
-    }
-
-    return m_definitions;
-  }
-}

ambari-server/src/main/java/org/apache/ambari/server/metadata/AmbariServiceAlertDefinitions.java

@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.metadata;
+
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.ambari.server.controller.RootServiceResponseFactory.Components;
+import org.apache.ambari.server.controller.RootServiceResponseFactory.Services;
+import org.apache.ambari.server.state.alert.AlertDefinition;
+import org.apache.ambari.server.state.alert.AlertDefinitionFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+/**
+ * The {@link AmbariServiceAlertDefinitions} class is used to represent the
+ * alerts defined in {@code alerts.json} which are for
+ * {@link Components#AMBARI_AGENT} and {@link Components#AMBARI_SERVER}. These
+ * alerts are bound to the host and are not part of a cluster or hadoop service.
+ */
+@Singleton
+public class AmbariServiceAlertDefinitions {
+
+  /**
+   * Logger.
+   */
+  private final static Logger LOG = LoggerFactory.getLogger(AmbariServiceAlertDefinitions.class);
+
+  /**
+   * The agent host definitions.
+   */
+  private List<AlertDefinition> m_agentDefinitions = null;
+
+  /**
+   * The server definitions.
+   */
+  private List<AlertDefinition> m_serverDefinitions = null;
+
+  /**
+   * The factory that will load the definitions from the alerts.json file.
+   */
+  @Inject
+  private AlertDefinitionFactory m_factory;
+
+  /**
+   * Gets all of the {@link AlertDefinition}s that exist on the path for all
+   * agent hosts.
+   *
+   * @return the alerts with {@link Components#AMBARI_AGENT} as the component
+   *         and {@code AMBARI} as the service.
+   */
+  public List<AlertDefinition> getAgentDefinitions() {
+    if (null != m_agentDefinitions) {
+      return m_agentDefinitions;
+    }
+
+    m_agentDefinitions = getDefinitions(Components.AMBARI_AGENT);
+    return m_agentDefinitions;
+  }
+
+  /**
+   * Gets all of the {@link AlertDefinition}s that exist on the path for
+   * {@link Components#AMBARI_SERVER}.
+   *
+   * @return the alerts with {@link Components#AMBARI_SERVER} as the component
+   *         and {@code AMBARI} as the service.
+   */
+  public List<AlertDefinition> getServerDefinitions() {
+    if (null != m_serverDefinitions) {
+      return m_serverDefinitions;
+    }
+
+    m_serverDefinitions = getDefinitions(Components.AMBARI_SERVER);
+    return m_serverDefinitions;
+  }
+
+  /**
+   * Loads the definitions for the {@code AMBARI} service for the specified
+   * component.
+   *
+   * @param component
+   *          the component (not {@code null}).
+   * @return the alert definitions for {@code AMBARI} service for the given
+   *         component.
+   */
+  private List<AlertDefinition> getDefinitions(Components component) {
+    List<AlertDefinition> definitions = new ArrayList<AlertDefinition>();
+
+    InputStream inputStream = ClassLoader.getSystemResourceAsStream("alerts.json");
+    InputStreamReader reader = new InputStreamReader(inputStream);
+
+    try {
+      Set<AlertDefinition> allDefinitions = m_factory.getAlertDefinitions(
+          reader, Services.AMBARI.name());
+
+      String componentName = component.name();
+
+      for (AlertDefinition definition : allDefinitions) {
+        if (componentName.equals(definition.getComponentName())) {
+          definitions.add(definition);
+        }
+      }
+
+    } catch (Exception exception) {
+      LOG.error("Unable to load the Ambari alerts JSON file", exception);
+    }
+
+    return definitions;
+  }
+}

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java

@@ -285,7 +285,13 @@ public class AlertsDAO {
     // pagination
     TypedQuery<AlertCurrentEntity> typedQuery = entityManager.createQuery(query);
     if( null != request.Pagination ){
-      typedQuery.setFirstResult(request.Pagination.getOffset());
+      // prevent JPA errors when -1 is passed in by accident
+      int offset = request.Pagination.getOffset();
+      if (offset < 0) {
+        offset = 0;
+      }
+
+      typedQuery.setFirstResult(offset);
       typedQuery.setMaxResults(request.Pagination.getPageSize());
     }
 

ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertDefinitionFactory.java

@@ -328,6 +328,10 @@ public class AlertDefinitionFactory {
           clazz = WebSource.class;
           break;
         }
+        case SERVER:{
+          clazz = ServerSource.class;
+          break;
+        }
         default:
           break;
       }

ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertDefinitionHash.java

@@ -380,6 +380,11 @@ public class AlertDefinitionHash {
       return affectedHosts;
     }
 
+    // ignore other AMBARI components as they are server-side only
+    if (ambariServiceName.equalsIgnoreCase(definitionServiceName)) {
+      return Collections.emptySet();
+    }
+
     // find all hosts that have the matching service and component
     for (String hostName : hosts.keySet()) {
       List<ServiceComponentHost> hostComponents = cluster.getServiceComponentHosts(hostName);

ambari-server/src/main/java/org/apache/ambari/server/state/alert/ServerSource.java

@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.state.alert;
+
+import com.google.gson.annotations.SerializedName;
+
+
+/**
+ * Alert when the source type is defined as {@link SourceType#SERVER}
+ */
+public class ServerSource extends Source {
+
+  @SerializedName("class")
+  private String m_class;
+
+  /**
+   * Gets the fully qualified classname specified in the source.
+   */
+  public String getSourceClass() {
+    return m_class;
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = super.hashCode();
+    result = prime * result + ((m_class == null) ? 0 : m_class.hashCode());
+    return result;
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+
+    if (!super.equals(obj)) {
+      return false;
+    }
+
+    if (getClass() != obj.getClass()) {
+      return false;
+    }
+
+    ServerSource other = (ServerSource) obj;
+    if (m_class == null) {
+      if (other.m_class != null) {
+        return false;
+      }
+    } else if (!m_class.equals(other.m_class)) {
+      return false;
+    }
+    return true;
+  }
+
+}

ambari-server/src/main/java/org/apache/ambari/server/state/alert/SourceType.java

@@ -25,14 +25,17 @@ public enum SourceType {
    * Source is from metric data.
    */
   METRIC,
+
   /**
    * Source is generated using of a script
    */
   SCRIPT,
+
   /**
    * Source is a simple port check
    */
   PORT,
+
   /**
    * Source is an aggregate of a collection of other alert states
    */
@@ -46,5 +49,10 @@ public enum SourceType {
   /**
    * Source is an http(s)-style request.
    */
-  WEB;
+  WEB,
+
+  /**
+   * A server-side alert.
+   */
+  SERVER;
 }

ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java

@@ -0,0 +1,291 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.state.services;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.ambari.server.AmbariService;
+import org.apache.ambari.server.controller.RootServiceResponseFactory.Components;
+import org.apache.ambari.server.controller.RootServiceResponseFactory.Services;
+import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
+import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.alert.AlertDefinition;
+import org.apache.ambari.server.state.alert.AlertDefinitionFactory;
+import org.apache.ambari.server.state.alert.ServerSource;
+import org.apache.ambari.server.state.alert.SourceType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.util.concurrent.AbstractScheduledService;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.Provider;
+
+/**
+ * The {@link AmbariServerAlertService} is used to manage the dynamically loaded
+ */
+@AmbariService
+public class AmbariServerAlertService extends AbstractScheduledService {
+
+  /**
+   * Logger.
+   */
+  private final static Logger LOG = LoggerFactory.getLogger(AmbariServerAlertService.class);
+
+  /**
+   * Used to inject the constructed {@link Runnable}s.
+   */
+  @Inject
+  private Injector m_injector;
+
+  /**
+   * Used for looking up alert definitions.
+   */
+  @Inject
+  private AlertDefinitionDAO m_dao;
+
+  /**
+   * Used to get alert definitions to use when generating alert instances.
+   */
+  @Inject
+  private Provider<Clusters> m_clustersProvider;
+
+  /**
+   * Used to coerce {@link AlertDefinitionEntity} into {@link AlertDefinition}.
+   */
+  @Inject
+  private AlertDefinitionFactory m_alertDefinitionFactory;
+
+  /**
+   * The executor to use to run all {@link Runnable} alert classes.
+   */
+  private final ScheduledExecutorService m_scheduledExecutorService = Executors.newScheduledThreadPool(3);
+
+  /**
+   * A map of all of the definition names to {@link ScheduledFuture}s.
+   */
+  private final Map<String, ScheduledAlert> m_futureMap = new ConcurrentHashMap<String, ScheduledAlert>();
+
+  /**
+   * Constructor.
+   *
+   */
+  public AmbariServerAlertService() {
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  protected Scheduler scheduler() {
+    return Scheduler.newFixedDelaySchedule(1, 1, TimeUnit.MINUTES);
+  }
+
+  /**
+   * {@inheritDoc}
+   * <p/>
+   * Loads all of the {@link Components#AMBARI_SERVER} definitions and schedules
+   * the ones that are enabled.
+   */
+  @Override
+  protected void startUp() throws Exception {
+    Map<String, Cluster> clusterMap = m_clustersProvider.get().getClusters();
+    for (Cluster cluster : clusterMap.values()) {
+      List<AlertDefinitionEntity> entities = m_dao.findByServiceComponent(
+          cluster.getClusterId(), Services.AMBARI.name(),
+          Components.AMBARI_SERVER.name());
+
+      for (AlertDefinitionEntity entity : entities) {
+        // don't schedule disabled alert definitions
+        if (!entity.getEnabled()) {
+          continue;
+        }
+
+        SourceType sourceType = entity.getSourceType();
+        if (sourceType != SourceType.SERVER) {
+          continue;
+        }
+
+        // schedule the Runnable for the definition
+        scheduleRunnable(entity);
+      }
+    }
+  }
+
+  /**
+   * {@inheritDoc}
+   * <p/>
+   * Compares all known {@link Components#AMBARI_SERVER} alerts with those that
+   * are scheduled. If any are not scheduled or have their intervals changed,
+   * then reschedule those.
+   */
+  @Override
+  protected void runOneIteration() throws Exception {
+    Map<String, Cluster> clusterMap = m_clustersProvider.get().getClusters();
+    for (Cluster cluster : clusterMap.values()) {
+      // get all of the cluster alerts for the server
+      List<AlertDefinitionEntity> entities = m_dao.findByServiceComponent(
+          cluster.getClusterId(), Services.AMBARI.name(),
+          Components.AMBARI_SERVER.name());
+
+      // for each alert, check to see if it's scheduled correctly
+      for (AlertDefinitionEntity entity : entities) {
+        String definitionName = entity.getDefinitionName();
+        ScheduledAlert scheduledAlert = m_futureMap.get(definitionName);
+        ScheduledFuture<?> scheduledFuture = scheduledAlert.getScheduledFuture();
+
+        // if the definition is not enabled, ensure it's not scheduled and
+        // then continue to the next one
+        if (!entity.getEnabled()) {
+          unschedule(definitionName, scheduledFuture);
+          continue;
+        }
+
+        // if there is no future, then schedule it
+        if (null == scheduledFuture) {
+          scheduleRunnable(entity);
+          continue;
+        }
+
+        // compare the delay of the future to the definition; if they don't
+        // match then reschedule this definition
+        int scheduledInterval = scheduledAlert.getInterval();
+        if (scheduledInterval != entity.getScheduleInterval()) {
+          // unschedule
+          unschedule(definitionName, scheduledFuture);
+
+          // reschedule
+          scheduleRunnable(entity);
+        }
+      }
+    }
+  }
+
+  /**
+   * Invokes {@link ScheduledFuture#cancel(boolean)} and removes the mapping
+   * from {@link #m_futureMap}. Does nothing if the future is not scheduled.
+   *
+   * @param scheduledFuture
+   */
+  private void unschedule(String definitionName,
+      ScheduledFuture<?> scheduledFuture) {
+    scheduledFuture.cancel(true);
+    m_futureMap.remove(definitionName);
+
+    LOG.info("Unscheduled server alert {}", definitionName);
+  }
+
+  /**
+   * Schedules the {@link Runnable} referenced by the
+   * {@link AlertDefinitionEntity} to run at a fixed interval.
+   *
+   * @param entity
+   *          the entity to schedule the runnable for (not {@code null}).
+   * @throws ClassNotFoundException
+   * @throws IllegalAccessException
+   * @throws InstantiationException
+   */
+  private void scheduleRunnable(AlertDefinitionEntity entity)
+      throws ClassNotFoundException,
+      IllegalAccessException, InstantiationException {
+
+    // if the definition is disabled, do nothing
+    if (!entity.getEnabled()) {
+      return;
+    }
+
+    AlertDefinition definition = m_alertDefinitionFactory.coerce(entity);
+    ServerSource serverSource = (ServerSource) definition.getSource();
+    String sourceClass = serverSource.getSourceClass();
+    int interval = definition.getInterval();
+
+    try {
+      Class<?> clazz = Class.forName(sourceClass);
+      if (!Runnable.class.isAssignableFrom(clazz)) {
+        LOG.warn(
+            "Unable to schedule a server side alert for {} because it is not a Runnable",
+            sourceClass);
+        return;
+      }
+
+      // instantiate and inject
+      Runnable runnable = (Runnable) clazz.newInstance();
+      m_injector.injectMembers(runnable);
+
+      // schedule the runnable alert
+      ScheduledFuture<?> scheduledFuture = m_scheduledExecutorService.scheduleWithFixedDelay(
+          runnable, interval, interval, TimeUnit.MINUTES);
+
+      String definitionName = entity.getDefinitionName();
+      ScheduledAlert scheduledAlert = new ScheduledAlert(scheduledFuture, interval);
+      m_futureMap.put(definitionName, scheduledAlert);
+
+      LOG.info("Scheduled server alert {} to run every {} minutes",
+          definitionName, interval);
+
+    } catch (ClassNotFoundException cnfe) {
+      LOG.warn(
+          "Unable to schedule a server side alert for {} because it could not be found in the classpath",
+          sourceClass);
+    }
+  }
+
+  /**
+   * The {@link ScheduledAlert} class is used as a way to encapsulate a
+   * {@link ScheduledFuture} with the interval it was scheduled with.
+   */
+  private static final class ScheduledAlert {
+    private final ScheduledFuture<?> m_scheduledFuture;
+    private final int m_interval;
+
+
+    /**
+     * Constructor.
+     *
+     * @param scheduledFuture
+     * @param interval
+     */
+    private ScheduledAlert(ScheduledFuture<?> scheduledFuture, int interval) {
+
+      m_scheduledFuture = scheduledFuture;
+      m_interval = interval;
+    }
+
+    /**
+     * @return the scheduledFuture
+     */
+    private ScheduledFuture<?> getScheduledFuture() {
+      return m_scheduledFuture;
+    }
+
+    /**
+     * @return the interval
+     */
+    private int getInterval() {
+      return m_interval;
+    }
+  }
+}

ambari-server/src/main/resources/alerts.json

@@ -2,6 +2,32 @@
   "AMBARI": {
     "service": [
     ],
+    "AMBARI_SERVER" : [
+      {
+        "name": "ambari_server_agent_heartbeat",
+        "label": "Ambari Agent Heartbeat",
+        "description": "This alert is triggered if the server has lost contact with an agent.",
+        "interval": 2,
+        "scope": "HOST",
+        "enabled": true,
+        "source": {
+          "type": "SERVER",
+          "class": "org.apache.ambari.server.alerts.AgentHeartbeatAlertRunnable"
+        }
+      },
+      {
+        "name": "ambari_server_stale_alerts",
+        "label": "Ambari Server Alerts",
+        "description": "This alert is triggered if the server detects that there are alerts which have not run in a timely manner.",
+        "interval": 5,
+        "scope": "SERVICE",
+        "enabled": true,
+        "source": {
+          "type": "SERVER",
+          "class": "org.apache.ambari.server.alerts.StaleAlertRunnable"
+        }
+      }      
+    ],
     "AMBARI_AGENT" : [
       {
         "name": "ambari_agent_disk_usage",

ambari-server/src/test/java/org/apache/ambari/server/alerts/AgentHeartbeatAlertRunnableTest.java

@@ -0,0 +1,228 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.alerts;
+
+import static junit.framework.Assert.assertEquals;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.persistence.EntityManager;
+
+import org.apache.ambari.server.events.AlertEvent;
+import org.apache.ambari.server.events.AlertReceivedEvent;
+import org.apache.ambari.server.events.MockEventListener;
+import org.apache.ambari.server.events.publishers.AlertEventPublisher;
+import org.apache.ambari.server.orm.DBAccessor;
+import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
+import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
+import org.apache.ambari.server.state.Alert;
+import org.apache.ambari.server.state.AlertState;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Host;
+import org.apache.ambari.server.state.HostState;
+import org.easymock.EasyMock;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.eventbus.EventBus;
+import com.google.inject.Binder;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+
+/**
+ * Tests {@link AgentHeartbeatAlertRunnable}.
+ */
+public class AgentHeartbeatAlertRunnableTest {
+
+  private final static long CLUSTER_ID = 1;
+  private final static String CLUSTER_NAME = "c1";
+  private final static String HOSTNAME = "c6401.ambari.apache.org";
+
+  private final static String DEFINITION_NAME = "ambari_server_agent_heartbeat";
+  private final static String DEFINITION_SERVICE = "AMBARI";
+  private final static String DEFINITION_COMPONENT = "AMBARI_SERVER";
+  private final static String DEFINITION_LABEL = "Mock Definition";
+
+  private Clusters m_clusters;
+  private Cluster m_cluster;
+  private Host m_host;
+  private Injector m_injector;
+  private AlertDefinitionDAO m_definitionDao;
+  private AlertDefinitionEntity m_definition;
+  private MockEventListener m_listener;
+
+  private AlertEventPublisher m_eventPublisher;
+  private EventBus m_synchronizedBus;
+
+  /**
+   *
+   */
+  @Before
+  public void setup() throws Exception {
+    m_injector = Guice.createInjector(new MockModule());
+    m_definitionDao = m_injector.getInstance(AlertDefinitionDAO.class);
+    m_clusters = m_injector.getInstance(Clusters.class);
+    m_cluster = m_injector.getInstance(Cluster.class);
+    m_eventPublisher = m_injector.getInstance(AlertEventPublisher.class);
+    m_listener = m_injector.getInstance(MockEventListener.class);
+    m_definition = EasyMock.createNiceMock(AlertDefinitionEntity.class);
+    m_host = EasyMock.createNiceMock(Host.class);
+
+    // !!! need a synchronous op for testing
+    m_synchronizedBus = new EventBus();
+    Field field = AlertEventPublisher.class.getDeclaredField("m_eventBus");
+    field.setAccessible(true);
+    field.set(m_eventPublisher, m_synchronizedBus);
+
+    // register mock listener
+    m_synchronizedBus.register(m_listener);
+
+    // create the cluster map
+    Map<String,Cluster> clusterMap = new HashMap<String, Cluster>();
+    clusterMap.put(CLUSTER_NAME, m_cluster);
+
+    // create the host map
+    Map<String, Host> hostMap = new HashMap<String, Host>();
+    hostMap.put(HOSTNAME, m_host);
+
+    // mock the definition for the alert
+    expect(m_definition.getDefinitionName()).andReturn(DEFINITION_NAME).atLeastOnce();
+    expect(m_definition.getServiceName()).andReturn(DEFINITION_SERVICE).atLeastOnce();
+    expect(m_definition.getComponentName()).andReturn(DEFINITION_COMPONENT).atLeastOnce();
+    expect(m_definition.getLabel()).andReturn(DEFINITION_LABEL).atLeastOnce();
+    expect(m_definition.getEnabled()).andReturn(true).atLeastOnce();
+
+    // mock the host state
+    expect(m_host.getState()).andReturn(HostState.HEALTHY);
+
+    // mock the cluster
+    expect(m_cluster.getClusterId()).andReturn(CLUSTER_ID).atLeastOnce();
+    expect(m_cluster.getClusterName()).andReturn(CLUSTER_NAME).atLeastOnce();
+
+    // mock clusters
+    expect(m_clusters.getClusters()).andReturn(clusterMap).atLeastOnce();
+    expect(m_clusters.getHostsForCluster(CLUSTER_NAME)).andReturn(hostMap).atLeastOnce();
+
+    // mock the definition DAO
+    expect(m_definitionDao.findByName(CLUSTER_ID, DEFINITION_NAME)).andReturn(
+        m_definition).atLeastOnce();
+
+    EasyMock.replay(m_definition, m_host, m_cluster, m_clusters,
+        m_definitionDao);
+    }
+
+  /**
+   * @throws Exception
+   */
+  @After
+  public void teardown() throws Exception {
+  }
+
+  @Test
+  public void testHealthyHostAlert(){
+    // precondition that no events were fired
+    assertEquals(0,
+        m_listener.getAlertEventReceivedCount(AlertReceivedEvent.class));
+
+    // instantiate and inject mocks
+    AgentHeartbeatAlertRunnable runnable = new AgentHeartbeatAlertRunnable();
+    m_injector.injectMembers(runnable);
+
+    // run the alert
+    runnable.run();
+
+    assertEquals(1,
+        m_listener.getAlertEventReceivedCount(AlertReceivedEvent.class));
+
+    List<AlertEvent> events = m_listener.getAlertEventInstances(AlertReceivedEvent.class);
+    assertEquals(1, events.size());
+
+    AlertReceivedEvent event = (AlertReceivedEvent) events.get(0);
+    Alert alert = event.getAlert();
+    assertEquals("AMBARI", alert.getService());
+    assertEquals("AMBARI_SERVER", alert.getComponent());
+    assertEquals(AlertState.OK, alert.getState());
+    assertEquals(DEFINITION_NAME, alert.getName());
+
+    verify(m_definition, m_host, m_cluster, m_clusters,
+        m_definitionDao);
+  }
+
+  @Test
+  public void testUnhealthyAlert() {
+    EasyMock.reset(m_host);
+    expect(m_host.getState()).andReturn(HostState.HEARTBEAT_LOST).atLeastOnce();
+    replay(m_host);
+
+    // precondition that no events were fired
+    assertEquals(0,
+        m_listener.getAlertEventReceivedCount(AlertReceivedEvent.class));
+
+    // instantiate and inject mocks
+    AgentHeartbeatAlertRunnable runnable = new AgentHeartbeatAlertRunnable();
+    m_injector.injectMembers(runnable);
+
+    // run the alert
+    runnable.run();
+
+    assertEquals(1,
+        m_listener.getAlertEventReceivedCount(AlertReceivedEvent.class));
+
+    List<AlertEvent> events = m_listener.getAlertEventInstances(AlertReceivedEvent.class);
+    assertEquals(1, events.size());
+
+    AlertReceivedEvent event = (AlertReceivedEvent) events.get(0);
+    Alert alert = event.getAlert();
+    assertEquals("AMBARI", alert.getService());
+    assertEquals("AMBARI_SERVER", alert.getComponent());
+    assertEquals(AlertState.CRITICAL, alert.getState());
+    assertEquals(DEFINITION_NAME, alert.getName());
+
+    verify(m_definition, m_host, m_cluster, m_clusters, m_definitionDao);
+  }
+
+  /**
+   *
+   */
+  private class MockModule implements Module {
+    /**
+     *
+     */
+    @Override
+    public void configure(Binder binder) {
+      Cluster cluster = EasyMock.createNiceMock(Cluster.class);
+
+      binder.bind(Clusters.class).toInstance(createNiceMock(Clusters.class));
+      binder.bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class));
+      binder.bind(Cluster.class).toInstance(cluster);
+      binder.bind(AlertDefinitionDAO.class).toInstance(createNiceMock(AlertDefinitionDAO.class));
+      binder.bind(EntityManager.class).toInstance(createNiceMock(EntityManager.class));
+    }
+  }
+}

ambari-server/src/test/java/org/apache/ambari/server/alerts/StaleAlertRunnableTest.java

@@ -0,0 +1,317 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.alerts;
+
+import static junit.framework.Assert.assertEquals;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.persistence.EntityManager;
+
+import org.apache.ambari.server.events.AlertEvent;
+import org.apache.ambari.server.events.AlertReceivedEvent;
+import org.apache.ambari.server.events.MockEventListener;
+import org.apache.ambari.server.events.publishers.AlertEventPublisher;
+import org.apache.ambari.server.orm.DBAccessor;
+import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
+import org.apache.ambari.server.orm.dao.AlertsDAO;
+import org.apache.ambari.server.orm.entities.AlertCurrentEntity;
+import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
+import org.apache.ambari.server.orm.entities.AlertHistoryEntity;
+import org.apache.ambari.server.state.Alert;
+import org.apache.ambari.server.state.AlertState;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.MaintenanceState;
+import org.easymock.EasyMock;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.eventbus.EventBus;
+import com.google.inject.Binder;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+
+/**
+ * Tests {@link StaleAlertRunnableTest}.
+ */
+public class StaleAlertRunnableTest {
+
+  private final static long CLUSTER_ID = 1;
+  private final static String CLUSTER_NAME = "c1";
+
+  private final static String DEFINITION_NAME = "ambari_server_stale_alerts";
+  private final static String DEFINITION_SERVICE = "AMBARI";
+  private final static String DEFINITION_COMPONENT = "AMBARI_SERVER";
+  private final static String DEFINITION_LABEL = "Mock Definition";
+  private final static int DEFINITION_INTERVAL = 1;
+
+  private Clusters m_clusters;
+  private Cluster m_cluster;
+  private Injector m_injector;
+  private AlertsDAO m_alertsDao;
+  private AlertDefinitionDAO m_definitionDao;
+  private AlertDefinitionEntity m_definition;
+  private List<AlertCurrentEntity> m_currentAlerts = new ArrayList<AlertCurrentEntity>();
+  private MockEventListener m_listener;
+
+  private AlertEventPublisher m_eventPublisher;
+  private EventBus m_synchronizedBus;
+
+  /**
+   *
+   */
+  @Before
+  public void setup() throws Exception {
+    m_injector = Guice.createInjector(new MockModule());
+    m_alertsDao = m_injector.getInstance(AlertsDAO.class);
+    m_definitionDao = m_injector.getInstance(AlertDefinitionDAO.class);
+    m_clusters = m_injector.getInstance(Clusters.class);
+    m_cluster = m_injector.getInstance(Cluster.class);
+    m_eventPublisher = m_injector.getInstance(AlertEventPublisher.class);
+    m_listener = m_injector.getInstance(MockEventListener.class);
+    m_definition = EasyMock.createNiceMock(AlertDefinitionEntity.class);
+
+    // !!! need a synchronous op for testing
+    m_synchronizedBus = new EventBus();
+    Field field = AlertEventPublisher.class.getDeclaredField("m_eventBus");
+    field.setAccessible(true);
+    field.set(m_eventPublisher, m_synchronizedBus);
+
+    // register mock listener
+    m_synchronizedBus.register(m_listener);
+
+    // create the cluster map
+    Map<String,Cluster> clusterMap = new HashMap<String, Cluster>();
+    clusterMap.put(CLUSTER_NAME, m_cluster);
+
+    // mock the definition for the alert
+    expect(m_definition.getDefinitionName()).andReturn(DEFINITION_NAME).atLeastOnce();
+    expect(m_definition.getServiceName()).andReturn(DEFINITION_SERVICE).atLeastOnce();
+    expect(m_definition.getComponentName()).andReturn(DEFINITION_COMPONENT).atLeastOnce();
+    expect(m_definition.getLabel()).andReturn(DEFINITION_LABEL).atLeastOnce();
+    expect(m_definition.getEnabled()).andReturn(true).atLeastOnce();
+    expect(m_definition.getScheduleInterval()).andReturn(DEFINITION_INTERVAL).atLeastOnce();
+
+    // mock the cluster
+    expect(m_cluster.getClusterId()).andReturn(CLUSTER_ID).atLeastOnce();
+
+    // mock clusters
+    expect(m_clusters.getClusters()).andReturn(clusterMap).atLeastOnce();
+
+    // mock the definition DAO
+    expect(m_definitionDao.findByName(CLUSTER_ID, DEFINITION_NAME)).andReturn(
+        m_definition).atLeastOnce();
+
+    // mock the current dao
+    expect(m_alertsDao.findCurrentByCluster(CLUSTER_ID)).andReturn(
+        m_currentAlerts).atLeastOnce();
+
+    replay(m_definition, m_cluster, m_clusters,
+        m_definitionDao, m_alertsDao);
+    }
+
+  /**
+   * @throws Exception
+   */
+  @After
+  public void teardown() throws Exception {
+  }
+
+  /**
+   * Tests that the event is triggerd with a status of OK.
+   */
+  @Test
+  public void testAllAlertsAreCurrent() {
+    // create current alerts that are not stale
+    AlertCurrentEntity current1 = createNiceMock(AlertCurrentEntity.class);
+    AlertHistoryEntity history1 = createNiceMock(AlertHistoryEntity.class);
+
+    expect(current1.getAlertHistory()).andReturn(history1).atLeastOnce();
+    expect(history1.getAlertDefinition()).andReturn(m_definition).atLeastOnce();
+
+    expect(current1.getMaintenanceState()).andReturn(MaintenanceState.OFF).atLeastOnce();
+    expect(current1.getLatestTimestamp()).andReturn(System.currentTimeMillis()).atLeastOnce();
+
+    replay(current1, history1);
+
+    m_currentAlerts.add(current1);
+
+    // precondition that no events were fired
+    assertEquals(0,
+        m_listener.getAlertEventReceivedCount(AlertReceivedEvent.class));
+
+    // instantiate and inject mocks
+    StaleAlertRunnable runnable = new StaleAlertRunnable();
+    m_injector.injectMembers(runnable);
+
+    // run the alert
+    runnable.run();
+
+    assertEquals(1,
+        m_listener.getAlertEventReceivedCount(AlertReceivedEvent.class));
+
+    List<AlertEvent> events = m_listener.getAlertEventInstances(AlertReceivedEvent.class);
+    assertEquals(1, events.size());
+
+    AlertReceivedEvent event = (AlertReceivedEvent) events.get(0);
+    Alert alert = event.getAlert();
+    assertEquals("AMBARI", alert.getService());
+    assertEquals("AMBARI_SERVER", alert.getComponent());
+    assertEquals(AlertState.OK, alert.getState());
+    assertEquals(DEFINITION_NAME, alert.getName());
+
+    verify(m_definition, m_cluster, m_clusters,
+        m_definitionDao);
+  }
+
+  /**
+   * Tests that a stale alert triggers the event with a status of CRITICAL.
+   */
+  @Test
+  public void testStaleAlert() {
+    // create current alerts that are stale
+    AlertCurrentEntity current1 = createNiceMock(AlertCurrentEntity.class);
+    AlertHistoryEntity history1 = createNiceMock(AlertHistoryEntity.class);
+
+    expect(current1.getAlertHistory()).andReturn(history1).atLeastOnce();
+    expect(history1.getAlertDefinition()).andReturn(m_definition).atLeastOnce();
+
+    // a really old timestampt to trigger the alert
+    expect(current1.getMaintenanceState()).andReturn(MaintenanceState.OFF).atLeastOnce();
+    expect(current1.getLatestTimestamp()).andReturn(1L).atLeastOnce();
+
+    replay(current1, history1);
+
+    m_currentAlerts.add(current1);
+
+    // precondition that no events were fired
+    assertEquals(0,
+        m_listener.getAlertEventReceivedCount(AlertReceivedEvent.class));
+
+    // instantiate and inject mocks
+    StaleAlertRunnable runnable = new StaleAlertRunnable();
+    m_injector.injectMembers(runnable);
+
+    // run the alert
+    runnable.run();
+
+    assertEquals(1,
+        m_listener.getAlertEventReceivedCount(AlertReceivedEvent.class));
+
+    List<AlertEvent> events = m_listener.getAlertEventInstances(AlertReceivedEvent.class);
+    assertEquals(1, events.size());
+
+    AlertReceivedEvent event = (AlertReceivedEvent) events.get(0);
+    Alert alert = event.getAlert();
+    assertEquals("AMBARI", alert.getService());
+    assertEquals("AMBARI_SERVER", alert.getComponent());
+    assertEquals(AlertState.CRITICAL, alert.getState());
+    assertEquals(DEFINITION_NAME, alert.getName());
+
+    verify(m_definition, m_cluster, m_clusters, m_definitionDao);
+  }
+
+  /**
+   * Tests that a stale alert in maintenance mode doesn't trigger the event.
+   */
+  @Test
+  public void testStaleAlertInMaintenaceMode() {
+    // create current alerts where 1 is stale but in maintence mode
+    AlertCurrentEntity current1 = createNiceMock(AlertCurrentEntity.class);
+    AlertHistoryEntity history1 = createNiceMock(AlertHistoryEntity.class);
+    AlertCurrentEntity current2 = createNiceMock(AlertCurrentEntity.class);
+    AlertHistoryEntity history2 = createNiceMock(AlertHistoryEntity.class);
+
+    expect(current1.getAlertHistory()).andReturn(history1).atLeastOnce();
+    expect(history1.getAlertDefinition()).andReturn(m_definition).atLeastOnce();
+
+    expect(current2.getAlertHistory()).andReturn(history2).atLeastOnce();
+    expect(history2.getAlertDefinition()).andReturn(m_definition).atLeastOnce();
+
+    // maintenance mode with a really old timestamp
+    expect(current1.getMaintenanceState()).andReturn(MaintenanceState.ON).atLeastOnce();
+    expect(current1.getLatestTimestamp()).andReturn(1L).atLeastOnce();
+
+    // an that that is not stale
+    expect(current2.getMaintenanceState()).andReturn(MaintenanceState.OFF).atLeastOnce();
+    expect(current2.getLatestTimestamp()).andReturn(System.currentTimeMillis()).atLeastOnce();
+
+    replay(current1, history1, current2, history2);
+
+    m_currentAlerts.add(current1);
+    m_currentAlerts.add(current2);
+
+    // precondition that no events were fired
+    assertEquals(0,
+        m_listener.getAlertEventReceivedCount(AlertReceivedEvent.class));
+
+    // instantiate and inject mocks
+    StaleAlertRunnable runnable = new StaleAlertRunnable();
+    m_injector.injectMembers(runnable);
+
+    // run the alert
+    runnable.run();
+
+    assertEquals(1,
+        m_listener.getAlertEventReceivedCount(AlertReceivedEvent.class));
+
+    List<AlertEvent> events = m_listener.getAlertEventInstances(AlertReceivedEvent.class);
+    assertEquals(1, events.size());
+
+    AlertReceivedEvent event = (AlertReceivedEvent) events.get(0);
+    Alert alert = event.getAlert();
+    assertEquals("AMBARI", alert.getService());
+    assertEquals("AMBARI_SERVER", alert.getComponent());
+    assertEquals(AlertState.OK, alert.getState());
+    assertEquals(DEFINITION_NAME, alert.getName());
+
+    verify(m_definition, m_cluster, m_clusters, m_definitionDao);
+  }
+
+  /**
+   *
+   */
+  private class MockModule implements Module {
+    /**
+     *
+     */
+    @Override
+    public void configure(Binder binder) {
+      Cluster cluster = EasyMock.createNiceMock(Cluster.class);
+
+      binder.bind(Clusters.class).toInstance(createNiceMock(Clusters.class));
+      binder.bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class));
+      binder.bind(Cluster.class).toInstance(cluster);
+      binder.bind(AlertDefinitionDAO.class).toInstance(createNiceMock(AlertDefinitionDAO.class));
+      binder.bind(AlertsDAO.class).toInstance(createNiceMock(AlertsDAO.class));
+      binder.bind(EntityManager.class).toInstance(createNiceMock(EntityManager.class));
+    }
+  }
+}

ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java

@@ -35,7 +35,6 @@ import java.lang.reflect.Field;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
@@ -51,7 +50,7 @@ import org.apache.ambari.server.StackAccessException;
 import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
 import org.apache.ambari.server.metadata.ActionMetadata;
-import org.apache.ambari.server.metadata.AgentAlertDefinitions;
+import org.apache.ambari.server.metadata.AmbariServiceAlertDefinitions;
 import org.apache.ambari.server.orm.GuiceJpaInitializer;
 import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
 import org.apache.ambari.server.orm.OrmTestHelper;
@@ -65,7 +64,6 @@ import org.apache.ambari.server.state.AutoDeployInfo;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.ComponentInfo;
-import org.apache.ambari.server.state.ConfigHelper;
 import org.apache.ambari.server.state.CustomCommandDefinition;
 import org.apache.ambari.server.state.DependencyInfo;
 import org.apache.ambari.server.state.OperatingSystemInfo;
@@ -1723,9 +1721,9 @@ public class AmbariMetaInfoTest {
     f.setAccessible(true);
     f.set(metaInfo, injector.getInstance(AlertDefinitionDAO.class));
 
-    f = c.getDeclaredField("agentAlertDefinitions");
+    f = c.getDeclaredField("ambariServiceAlertDefinitions");
     f.setAccessible(true);
-    f.set(metaInfo, injector.getInstance(AgentAlertDefinitions.class));
+    f.set(metaInfo, injector.getInstance(AmbariServiceAlertDefinitions.class));
 
     Clusters clusters = injector.getInstance(Clusters.class);
     Cluster cluster = clusters.getClusterById(clusterId);
@@ -1738,7 +1736,7 @@ public class AmbariMetaInfoTest {
 
     AlertDefinitionDAO dao = injector.getInstance(AlertDefinitionDAO.class);
     List<AlertDefinitionEntity> definitions = dao.findAll(clusterId);
-    assertEquals(7, definitions.size());
+    assertEquals(9, definitions.size());
 
     // figure out how many of these alerts were merged into from the
     // non-stack alerts.json
@@ -1751,7 +1749,7 @@ public class AmbariMetaInfoTest {
     }
 
     assertEquals(1, hostAlertCount);
-    assertEquals(6, definitions.size() - hostAlertCount);
+    assertEquals(8, definitions.size() - hostAlertCount);
 
     for (AlertDefinitionEntity definition : definitions) {
       definition.setScheduleInterval(28);
@@ -1761,7 +1759,7 @@ public class AmbariMetaInfoTest {
     metaInfo.reconcileAlertDefinitions(clusters);
 
     definitions = dao.findAll();
-    assertEquals(7, definitions.size());
+    assertEquals(9, definitions.size());
 
     for (AlertDefinitionEntity definition : definitions) {
       assertEquals(28, definition.getScheduleInterval().intValue());
@@ -1770,7 +1768,7 @@ public class AmbariMetaInfoTest {
     // find all enabled for the cluster should find 6 (the ones from HDFS;
     // it will not find the agent alert since it's not bound to the cluster)
     definitions = dao.findAllEnabled(cluster.getClusterId());
-    assertEquals(6, definitions.size());
+    assertEquals(8, definitions.size());
 
     // create new definition
     AlertDefinitionEntity entity = new AlertDefinitionEntity();
@@ -1789,18 +1787,19 @@ public class AmbariMetaInfoTest {
 
     // verify the new definition is found (6 HDFS + 1 new one)
     definitions = dao.findAllEnabled(cluster.getClusterId());
-    assertEquals(7, definitions.size());
+    assertEquals(9, definitions.size());
 
     // reconcile, which should disable our bad definition
     metaInfo.reconcileAlertDefinitions(clusters);
 
     // find all enabled for the cluster should find 6
     definitions = dao.findAllEnabled(cluster.getClusterId());
-    assertEquals(6, definitions.size());
+    assertEquals(8, definitions.size());
 
-    // find all should find 6 HDFS + 1 disabled + 1 agent alert
+    // find all should find 6 HDFS + 1 disabled + 1 agent alert + 2 server
+    // alerts
     definitions = dao.findAll();
-    assertEquals(8, definitions.size());
+    assertEquals(10, definitions.size());
 
     entity = dao.findById(entity.getDefinitionId());
     assertFalse(entity.getEnabled());

ambari-server/src/test/java/org/apache/ambari/server/events/MockEventListener.java

@@ -18,6 +18,7 @@
 package org.apache.ambari.server.events;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -37,13 +38,13 @@ public class MockEventListener {
    * When an event is received, its class is captured and the event object is
    * added to the list.
    */
-  private final Map<Class<?>, List<Object>> m_receivedAmbariEvents = new HashMap<Class<?>, List<Object>>();
+  private final Map<Class<?>, List<AmbariEvent>> m_receivedAmbariEvents = new HashMap<Class<?>, List<AmbariEvent>>();
 
   /**
    * When an event is received, its class is captured and the event object is
    * added to the list.
    */
-  private final Map<Class<?>, List<Object>> m_receivedAlertEvents = new HashMap<Class<?>, List<Object>>();
+  private final Map<Class<?>, List<AlertEvent>> m_receivedAlertEvents = new HashMap<Class<?>, List<AlertEvent>>();
 
   /**
    * Resets the captured events.
@@ -109,14 +110,43 @@ public class MockEventListener {
     return m_receivedAlertEvents.get(clazz).size();
   }
 
+
+  /**
+   * Get the instances
+   *
+   * @param clazz
+   * @return
+   */
+  public List<AmbariEvent> getAmbariEventInstances(Class<? extends AmbariEvent> clazz){
+    if (!m_receivedAmbariEvents.containsKey(clazz)) {
+      return Collections.emptyList();
+    }
+
+    return m_receivedAmbariEvents.get(clazz);
+  }
+
+  /**
+   * Get the instances
+   *
+   * @param clazz
+   * @return
+   */
+  public List<AlertEvent> getAlertEventInstances(Class<? extends AlertEvent> clazz){
+    if (!m_receivedAlertEvents.containsKey(clazz)) {
+      return Collections.emptyList();
+    }
+
+    return m_receivedAlertEvents.get(clazz);
+  }
+
   /**
    * @param event
    */
   @Subscribe
   public void onAmbariEvent(AmbariEvent event) {
-    List<Object> events = m_receivedAmbariEvents.get(event.getClass());
+    List<AmbariEvent> events = m_receivedAmbariEvents.get(event.getClass());
     if (null == events) {
-      events = new ArrayList<Object>();
+      events = new ArrayList<AmbariEvent>();
       m_receivedAmbariEvents.put(event.getClass(), events);
     }
 
@@ -128,9 +158,9 @@ public class MockEventListener {
    */
   @Subscribe
   public void onAlertEvent(AlertEvent event) {
-    List<Object> events = m_receivedAlertEvents.get(event.getClass());
+    List<AlertEvent> events = m_receivedAlertEvents.get(event.getClass());
     if (null == events) {
-      events = new ArrayList<Object>();
+      events = new ArrayList<AlertEvent>();
       m_receivedAlertEvents.put(event.getClass(), events);
     }
 

ambari-server/src/test/java/org/apache/ambari/server/metadata/AgentAlertDefinitionsTest.java

@@ -31,7 +31,7 @@ import com.google.inject.Guice;
 import com.google.inject.Injector;
 
 /**
- * Tets {@link AgentAlertDefinitions}.
+ * Tets {@link AmbariServiceAlertDefinitions}.
  */
 public class AgentAlertDefinitionsTest {
 
@@ -46,9 +46,9 @@ public class AgentAlertDefinitionsTest {
    * Tests loading the agent alerts.
    */
   @Test
-  public void testLoadingAlerts() {
-    AgentAlertDefinitions agentAlerts = m_injector.getInstance(AgentAlertDefinitions.class);
-    List<AlertDefinition> definitions = agentAlerts.getDefinitions();
+  public void testLoadingAgentHostAlerts() {
+    AmbariServiceAlertDefinitions ambariServiceAlertDefinitions = m_injector.getInstance(AmbariServiceAlertDefinitions.class);
+    List<AlertDefinition> definitions = ambariServiceAlertDefinitions.getAgentDefinitions();
     Assert.assertEquals(1, definitions.size());
 
     for( AlertDefinition definition : definitions){
@@ -58,4 +58,21 @@ public class AgentAlertDefinitionsTest {
       Assert.assertEquals("AMBARI", definition.getServiceName());
     }
   }
+
+  /**
+   * Tests loading the agent alerts.
+   */
+  @Test
+  public void testLoadingServertAlerts() {
+    AmbariServiceAlertDefinitions ambariServiceAlertDefinitions = m_injector.getInstance(AmbariServiceAlertDefinitions.class);
+    List<AlertDefinition> definitions = ambariServiceAlertDefinitions.getServerDefinitions();
+    Assert.assertEquals(2, definitions.size());
+
+    for (AlertDefinition definition : definitions) {
+      Assert.assertEquals(Components.AMBARI_SERVER.name(),
+          definition.getComponentName());
+
+      Assert.assertEquals("AMBARI", definition.getServiceName());
+    }
+  }
 }

ambari-web/app/controllers/main/alerts/definition_configs_controller.js

@@ -152,6 +152,9 @@ App.MainAlertDefinitionConfigsController = Em.Controller.extend({
       case 'AGGREGATE':
         configs = this.renderAggregateConfigs();
         break;
+      case 'SERVER':
+      	configs = this.renderServerConfigs();
+      	break;
       default:
         console.error('Incorrect Alert Definition Type: ', alertDefinitionType);
     }
@@ -348,6 +351,32 @@ App.MainAlertDefinitionConfigsController = Em.Controller.extend({
       })
     ];
   },
+  
+  /**
+   * Render config properties for server-type alert definition
+   * @method renderScriptConfigs
+   * @returns {App.AlertConfigProperty[]}
+   */
+  renderServerConfigs: function () {
+    var result = [];
+    var alertDefinition = this.get('content');
+    var isWizard = this.get('isWizard');
+
+    if (this.get('isWizard')) {
+      result = result.concat(this.renderCommonWizardConfigs());
+    }
+
+    result = result.concat([
+      App.AlertConfigProperties.Description.create({
+        value: isWizard ? '' : alertDefinition.get('description')
+      }),
+      App.AlertConfigProperties.Interval.create({
+        value: isWizard ? '' : alertDefinition.get('interval')
+      })
+    ]);
+
+    return result;
+  },  
 
   /**
    * Render common list of configs used in almost all alert types in wizard

ambari-web/app/mappers/alert_definitions_mapper.js

@@ -57,6 +57,9 @@ App.alertDefinitionsMapper = App.QuickDataMapper.create({
   scriptConfig: {
     location: 'AlertDefinition.source.path'
   },
+  
+  serverConfig: {
+  },
 
   uriConfig: {
     id: 'AlertDefinition.source.uri.id',
@@ -167,6 +170,9 @@ App.alertDefinitionsMapper = App.QuickDataMapper.create({
           case 'SCRIPT':
             alertDefinitions.push($.extend(alertDefinition, this.parseIt(item, this.get('scriptConfig'))));
             break;
+          case 'SERVER':
+            alertDefinitions.push($.extend(alertDefinition, this.parseIt(item, this.get('serverConfig'))));
+            break;
           default:
             console.error('Incorrect Alert Definition type:', item.AlertDefinition);
         }