AMBARI-15745 - Create Alert For Reporting Potential Issues With Slow REST Responses (part2) (jonathanhurley)

ambari-server/src/main/java/org/apache/ambari/server/alerts/AmbariPerformanceRunnable.java

@@ -20,25 +20,25 @@ package org.apache.ambari.server.alerts;
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashSet;
+import java.util.HashMap;
 import java.util.List;
-import java.util.Set;
+import java.util.Map;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.actionmanager.ActionManager;
 import org.apache.ambari.server.actionmanager.RequestStatus;
+import org.apache.ambari.server.api.query.Query;
+import org.apache.ambari.server.api.query.QueryImpl;
+import org.apache.ambari.server.api.query.render.DefaultRenderer;
+import org.apache.ambari.server.api.resources.ClusterResourceDefinition;
 import org.apache.ambari.server.api.services.BaseRequest;
-import org.apache.ambari.server.controller.AmbariManagementController;
-import org.apache.ambari.server.controller.internal.AbstractControllerResourceProvider;
-import org.apache.ambari.server.controller.internal.ClusterResourceProvider;
-import org.apache.ambari.server.controller.spi.Predicate;
-import org.apache.ambari.server.controller.spi.Request;
-import org.apache.ambari.server.controller.spi.Resource.Type;
-import org.apache.ambari.server.controller.spi.ResourceProvider;
-import org.apache.ambari.server.controller.utilities.PredicateBuilder;
-import org.apache.ambari.server.controller.utilities.PropertyHelper;
+import org.apache.ambari.server.controller.spi.ClusterController;
+import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.controller.utilities.ClusterControllerHelper;
 import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
+import org.apache.ambari.server.orm.dao.RequestDAO;
 import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
+import org.apache.ambari.server.security.authorization.internal.InternalAuthenticationToken;
 import org.apache.ambari.server.state.Alert;
 import org.apache.ambari.server.state.AlertState;
 import org.apache.ambari.server.state.Cluster;
@@ -51,9 +51,9 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.math.NumberUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.security.core.context.SecurityContextHolder;
 
 import com.google.inject.Inject;
-import com.google.inject.Provider;
 
 /**
  * The {@link AmbariPerformanceRunnable} is used by the
@@ -104,7 +104,7 @@ public class AmbariPerformanceRunnable extends AlertRunnable {
    * The {@link PerformanceArea} enumeration represents logical areas of
    * functionality to test for performance.
    */
-  private enum PerformanceArea {
+   enum PerformanceArea {
 
     /**
      * Query for requests by {@link RequestStatus#IN_PROGRESS}.
@@ -116,11 +116,31 @@ public class AmbariPerformanceRunnable extends AlertRunnable {
        */
       @Override
       void execute(AmbariPerformanceRunnable runnable, Cluster cluster) throws Exception {
-        runnable.m_actionManager.get().getRequestsByStatus(RequestStatus.IN_PROGRESS,
+        runnable.m_actionManager.getRequestsByStatus(RequestStatus.IN_PROGRESS,
             BaseRequest.DEFAULT_PAGE_SIZE, false);
       }
     },
 
+    /**
+     * Query for requests by {@link RequestStatus#IN_PROGRESS}.
+     */
+    HRC_SUMMARY_STATUS("Database Access (Task Status Aggregation)",
+        "task.status.aggregation.warning.threshold", 3000,
+        "task.status.aggregation.critical.threshold", 5000) {
+      /**
+       * {@inheritDoc}
+       */
+      @Override
+      void execute(AmbariPerformanceRunnable runnable, Cluster cluster) throws Exception {
+        List<Long> requestIds = runnable.m_requestDAO.findAllRequestIds(
+            BaseRequest.DEFAULT_PAGE_SIZE, false);
+
+        for (long requestId : requestIds) {
+          runnable.m_hostRoleCommandDAO.findAggregateCounts(requestId);
+        }
+      }
+    },
+
     /**
      * Query through the REST API framework for a cluster.
      */
@@ -132,25 +152,19 @@ public class AmbariPerformanceRunnable extends AlertRunnable {
        */
       @Override
       void execute(AmbariPerformanceRunnable runnable, Cluster cluster) throws Exception {
-        Type type = Type.Cluster;
-        ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider(
-            type, PropertyHelper.getPropertyIds(type), PropertyHelper.getKeyPropertyIds(type),
-            runnable.m_amc.get());
-
-        Set<String> propertyIds = new HashSet<String>();
-
-        propertyIds.add(ClusterResourceProvider.CLUSTER_ID_PROPERTY_ID);
-        propertyIds.add(ClusterResourceProvider.CLUSTER_NAME_PROPERTY_ID);
+        // Set authenticated user so that authorization checks will pass
+        InternalAuthenticationToken authenticationToken = new InternalAuthenticationToken("admin");
+        authenticationToken.setAuthenticated(true);
+        SecurityContextHolder.getContext().setAuthentication(authenticationToken);
 
         // create the request
-        Request request = PropertyHelper.getReadRequest(propertyIds);
+        Map<Resource.Type, String> mapIds = new HashMap<Resource.Type, String>();
+        mapIds.put(Resource.Type.Cluster, cluster.getClusterName());
 
-        // build the predicate for this cluster
-        Predicate predicate = new PredicateBuilder().property(
-            ClusterResourceProvider.CLUSTER_NAME_PROPERTY_ID).equals(
-                cluster.getClusterName()).toPredicate();
-
-        provider.getResources(request, predicate);
+        ClusterController clusterController = ClusterControllerHelper.getClusterController();
+        Query query = new QueryImpl(mapIds, new ClusterResourceDefinition(), clusterController);
+        query.setRenderer(new DefaultRenderer());
+        query.execute();
       }
     };
 
@@ -222,13 +236,22 @@ public class AmbariPerformanceRunnable extends AlertRunnable {
   }
 
   /**
-   * Used for querying for requests by status.
+   * Used for getting the most recent requests.
+   */
+  @Inject
+  private RequestDAO m_requestDAO;
+
+  /**
+   * Used for executing queries which are known to potentially take a long time.
    */
   @Inject
-  private Provider<ActionManager> m_actionManager;
+  private HostRoleCommandDAO m_hostRoleCommandDAO;
 
+  /**
+   * Used for querying for requests by status.
+   */
   @Inject
-  private Provider<AmbariManagementController> m_amc;
+  private ActionManager m_actionManager;
 
   /**
    * Constructor.

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java

@@ -23,7 +23,6 @@ import static org.apache.ambari.server.orm.dao.DaoUtils.ORACLE_LIST_LIMIT;
 
 import java.text.MessageFormat;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -318,7 +317,7 @@ public class HostRoleCommandDAO {
         "HostRoleCommandEntity.findByRequestIdAndStatuses", HostRoleCommandEntity.class);
     query.setParameter("requestId", requestId);
     query.setParameter("statuses", statuses);
-    List results = query.getResultList();
+    List<HostRoleCommandEntity> results = query.getResultList();
     return results;
   }
 
@@ -675,6 +674,7 @@ public class HostRoleCommandDAO {
    *          the request id
    * @return the map of stage-to-summary objects
    */
+  @RequiresSession
   public Map<Long, HostRoleCommandStatusSummaryDTO> findAggregateCounts(Long requestId) {
     if (!hostRoleCommandStatusSummaryCacheEnabled) {
       return loadAggregateCounts(requestId);
@@ -706,16 +706,17 @@ public class HostRoleCommandDAO {
    * @param requestId upgrade request id
    * @return Most recent task failure during stack upgrade, or null if one doesn't exist.
    */
+  @RequiresSession
   public HostRoleCommandEntity findMostRecentFailure(Long requestId) {
     TypedQuery<HostRoleCommandEntity> query = entityManagerProvider.get().createNamedQuery(
         "HostRoleCommandEntity.findTasksByStatusesOrderByIdDesc", HostRoleCommandEntity.class);
 
     query.setParameter("requestId", requestId);
     query.setParameter("statuses", HostRoleStatus.STACK_UPGRADE_FAILED_STATUSES);
-    List results = query.getResultList();
+    List<HostRoleCommandEntity> results = query.getResultList();
 
     if (!results.isEmpty()) {
-      HostRoleCommandEntity candidate = (HostRoleCommandEntity) results.get(0);
+      HostRoleCommandEntity candidate = results.get(0);
 
       // Ensure that there are no other completed tasks in a future stage to avoid returning an old error.
       // During Express Upgrade, we can run multiple commands in the same stage, so it's possible to have

ambari-server/src/main/resources/alerts.json

@@ -56,6 +56,24 @@
               "units": "ms",
               "threshold": "CRITICAL"
             },
+            {
+              "name": "task.status.aggregation.warning.threshold",
+              "display_name": "Warning",
+              "value": 3000,
+              "type": "NUMERIC",
+              "description": "The time to calculate a request's status from its tasks before a warning alert is triggered.",
+              "units": "ms",
+              "threshold": "WARNING"
+            },
+            {
+              "name": "task.status.aggregation.critical.threshold",
+              "display_name": "Critical",
+              "value": 5000,
+              "type": "NUMERIC",
+              "description": "The time to calculate a request's status from its tasks before a critical alert is triggered.",
+              "units": "ms",
+              "threshold": "CRITICAL"
+            },
             {
               "name": "rest.api.cluster.warning.threshold",
               "display_name": "Warning",

ambari-server/src/test/java/org/apache/ambari/server/alerts/AmbariPerformanceRunnableTest.java

@@ -19,6 +19,7 @@
 package org.apache.ambari.server.alerts;
 
 import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertTrue;
 import static org.easymock.EasyMock.createNiceMock;
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.replay;
@@ -33,8 +34,11 @@ import java.util.Map;
 import javax.persistence.EntityManager;
 
 import org.apache.ambari.server.actionmanager.ActionManager;
+import org.apache.ambari.server.alerts.AmbariPerformanceRunnable.PerformanceArea;
 import org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.controller.internal.ClusterResourceProvider;
+import org.apache.ambari.server.controller.spi.ClusterController;
+import org.apache.ambari.server.controller.utilities.ClusterControllerHelper;
 import org.apache.ambari.server.events.AlertEvent;
 import org.apache.ambari.server.events.AlertReceivedEvent;
 import org.apache.ambari.server.events.MockEventListener;
@@ -43,6 +47,7 @@ import org.apache.ambari.server.orm.DBAccessor;
 import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
 import org.apache.ambari.server.orm.dao.AlertsDAO;
 import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
+import org.apache.ambari.server.orm.dao.RequestDAO;
 import org.apache.ambari.server.orm.entities.AlertCurrentEntity;
 import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
 import org.apache.ambari.server.state.Alert;
@@ -57,6 +62,10 @@ import org.easymock.EasyMock;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.easymock.PowerMock;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
 
 import com.google.common.eventbus.EventBus;
 import com.google.inject.Binder;
@@ -67,6 +76,9 @@ import com.google.inject.Module;
 /**
  * Tests {@link AmbariPerformanceRunnable}.
  */
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ ClusterControllerHelper.class, AmbariPerformanceRunnable.class,
+    PerformanceArea.class })
 public class AmbariPerformanceRunnableTest {
 
   private final static long CLUSTER_ID = 1;
@@ -148,7 +160,13 @@ public class AmbariPerformanceRunnableTest {
     AlertDefinitionFactory factory = m_injector.getInstance(AlertDefinitionFactory.class);
     expect(factory.coerce(EasyMock.anyObject(AlertDefinitionEntity.class))).andReturn(definition).atLeastOnce();
 
-    replay(m_definition, m_cluster, m_clusters, m_definitionDao, m_alertsDao, factory);
+    ClusterController clusterController = m_injector.getInstance(ClusterController.class);
+    PowerMock.mockStatic(ClusterControllerHelper.class);
+    expect(ClusterControllerHelper.getClusterController()).andReturn(clusterController);
+    PowerMock.replay(ClusterControllerHelper.class);
+
+    replay(m_definition, m_cluster, m_clusters, m_definitionDao, m_alertsDao, factory,
+        clusterController);
     }
 
   /**
@@ -158,11 +176,53 @@ public class AmbariPerformanceRunnableTest {
   public void teardown() throws Exception {
   }
 
+  /**
+   * Tests that the event is triggerd with a status of OK if all performance
+   * areas pass.
+   */
+  @Test
+  public void testAlertFiresOKEvent() {
+    // mock the entire enum so that no problems are reported
+    PowerMock.mockStatic(PerformanceArea.class);
+    expect(PerformanceArea.values()).andReturn(new PerformanceArea[0]);
+    PowerMock.replay(PerformanceArea.class);
+
+    // instantiate and inject mocks
+    AmbariPerformanceRunnable runnable = new AmbariPerformanceRunnable(
+        m_definition.getDefinitionName());
+
+    m_injector.injectMembers(runnable);
+
+    // run the alert
+    runnable.run();
+
+    assertEquals(1, m_listener.getAlertEventReceivedCount(AlertReceivedEvent.class));
+
+    List<AlertEvent> events = m_listener.getAlertEventInstances(AlertReceivedEvent.class);
+    assertEquals(1, events.size());
+
+    AlertReceivedEvent event = (AlertReceivedEvent) events.get(0);
+    Alert alert = event.getAlert();
+    assertEquals("AMBARI", alert.getService());
+    assertEquals("AMBARI_SERVER", alert.getComponent());
+    assertEquals(AlertState.OK, alert.getState());
+    assertEquals(DEFINITION_NAME, alert.getName());
+
+    verify(m_cluster, m_clusters, m_definitionDao);
+  }
+
   /**
    * Tests that the event is triggerd with a status of UNKNOWN.
    */
   @Test
-  public void testAlertFiresEvents() {
+  public void testAlertFiresUnknownEvent() {
+    // mock one area, leaving others to fail
+    RequestDAO requestDAO = m_injector.getInstance(RequestDAO.class);
+    expect(requestDAO.findAllRequestIds(EasyMock.anyInt(), EasyMock.anyBoolean())).andReturn(
+        new ArrayList<Long>());
+
+    replay(requestDAO);
+
     // instantiate and inject mocks
     AmbariPerformanceRunnable runnable = new AmbariPerformanceRunnable(
         m_definition.getDefinitionName());
@@ -185,6 +245,10 @@ public class AmbariPerformanceRunnableTest {
     assertEquals(AlertState.UNKNOWN, alert.getState());
     assertEquals(DEFINITION_NAME, alert.getName());
 
+    // verify that even though there is 1 UNKNOWN, there should also be 1 OK as
+    // well
+    assertTrue(alert.getText().contains("(OK)"));
+
     verify(m_cluster, m_clusters, m_definitionDao);
   }
 
@@ -227,6 +291,8 @@ public class AmbariPerformanceRunnableTest {
       binder.bind(AmbariManagementController.class).toInstance(createNiceMock(AmbariManagementController.class));
       binder.bind(AlertDefinitionFactory.class).toInstance(createNiceMock(AlertDefinitionFactory.class));
       binder.bind(ClusterResourceProvider.class).toInstance(createNiceMock(ClusterResourceProvider.class));
+      binder.bind(ClusterController.class).toInstance(createNiceMock(ClusterController.class));
+      binder.bind(RequestDAO.class).toInstance(createNiceMock(RequestDAO.class));
     }
   }
 }