AMBARI-10065. Performance: ActionScheduler loads too many objects per Stage iteration (ncole)

ambari-server/src/main/java/org/apache/ambari/server/KdcServerConnectionVerification.java

@@ -45,11 +45,11 @@ import com.google.inject.Singleton;
  * <p>
  * It has two potential clients.
  * <ul>
- * <li>Ambari Agent: 
+ * <li>Ambari Agent:
  * 		Uses it to make sure host can talk to specified KDC Server
  * </li>
- * 
- * <li>Ambari Server: 
+ *
+ * <li>Ambari Server:
  * 		Uses it for connection check, like agent, and also validates
  * 		the credentials provided on Server side.
  * </li>
@@ -67,18 +67,18 @@ public class KdcServerConnectionVerification {
    * UDP connection timeout in seconds.
    */
   private int udpTimeout = 10;
-  
+
   @Inject
   public KdcServerConnectionVerification(Configuration config) {
     this.config = config;
   }
 
-  
+
   /**
    * Given server IP or hostname, checks if server is reachable i.e.
    * we can make a socket connection to it. Hostname may contain port
-   * number separated by a colon. 
-   * 
+   * number separated by a colon.
+   *
    * @param kdcHost KDC server IP or hostname (with optional port number)
    * @return true, if server is accepting connection given port; false otherwise.
    */
@@ -116,7 +116,7 @@ public class KdcServerConnectionVerification {
 
   /**
    * Attempt to connect to KDC server over TCP.
-   * 
+   *
    * @param server KDC server IP or hostname
    * @param port	 KDC server port
    * @return	true, if server is accepting connection given port; false otherwise.
@@ -181,7 +181,7 @@ public class KdcServerConnectionVerification {
       }
     });
 
-    new Thread(future).start();
+    new Thread(future, "ambari-kdc-verify").start();
     Boolean result;
     try {
       // timeout after specified timeout

ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessor.java

@@ -24,6 +24,7 @@ import java.util.Map;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.agent.CommandReport;
 import org.apache.ambari.server.agent.ExecutionCommand;
+import org.apache.ambari.server.orm.entities.RequestEntity;
 
 import com.google.inject.persist.Transactional;
 
@@ -40,9 +41,18 @@ public interface ActionDBAccessor {
   public List<Stage> getAllStages(long requestId);
 
   /**
-   * Get request object by id
-   * @param requestId
-   * @return
+   * Gets the request entity by id.  Will not load the entire
+   * Request/Stage/HostRoleCommand object hierarchy.
+   */
+  RequestEntity getRequestEntity(long requestId);
+
+  /**
+   * Get request object by id.  USE WITH CAUTION!  This method will
+   * result in loading the full object hierarchy, and can be expensive to
+   * construct.
+   * @param requestId the request id
+   * @return the Request instance, with all Stages and HostRoleCommands constructed
+   *        from the database
    */
   Request getRequest(long requestId);
 
@@ -205,4 +215,6 @@ public interface ActionDBAccessor {
    * Gets request objects by ids
    */
   public List<Request> getRequests(Collection<Long> requestIds);
+
+
 }

ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java

@@ -140,9 +140,14 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
     return stages;
   }
 
+  @Override
+  public RequestEntity getRequestEntity(long requestId) {
+    return requestDAO.findByPK(requestId);
+  }
+
   @Override
   public Request getRequest(long requestId) {
-    RequestEntity requestEntity = requestDAO.findByPK(requestId);
+    RequestEntity requestEntity = getRequestEntity(requestId);
     if (requestEntity != null) {
       return requestFactory.createExisting(requestEntity);
     } else {
@@ -309,7 +314,7 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
 
   @Override
   public void startRequest(long requestId) {
-    RequestEntity requestEntity = requestDAO.findByPK(requestId);
+    RequestEntity requestEntity = getRequestEntity(requestId);
     if (requestEntity != null && requestEntity.getStartTime() == -1L) {
       requestEntity.setStartTime(System.currentTimeMillis());
       requestDAO.merge(requestEntity);
@@ -318,7 +323,7 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
 
   @Override
   public void endRequest(long requestId) {
-    RequestEntity requestEntity = requestDAO.findByPK(requestId);
+    RequestEntity requestEntity = getRequestEntity(requestId);
     if (requestEntity != null && requestEntity.getEndTime() == -1L) {
       requestEntity.setEndTime(System.currentTimeMillis());
       requestDAO.merge(requestEntity);

ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java

@@ -44,6 +44,7 @@ import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.HostsMap;
 import org.apache.ambari.server.events.ActionFinalReportReceivedEvent;
 import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
+import org.apache.ambari.server.orm.entities.RequestEntity;
 import org.apache.ambari.server.serveraction.ServerActionExecutor;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
@@ -152,7 +153,7 @@ class ActionScheduler implements Runnable {
   }
 
   public void start() {
-    schedulerThread = new Thread(this);
+    schedulerThread = new Thread(this, "ambari-action-scheduler");
     schedulerThread.start();
 
     // Start up the ServerActionExecutor. Since it is directly related to the ActionScheduler it
@@ -252,7 +253,8 @@ class ActionScheduler implements Runnable {
         i_stage ++;
         long requestId = stage.getRequestId();
         LOG.debug("==> STAGE_i = " + i_stage + "(requestId=" + requestId + ",StageId=" + stage.getStageId() + ")");
-        Request request = db.getRequest(requestId);
+
+        RequestEntity request = db.getRequestEntity(requestId);
 
         if (request.isExclusive()) {
           if (runningRequestIds.size() > 0 ) {

ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java

@@ -94,7 +94,7 @@ public class HeartbeatMonitor implements Runnable {
   }
 
   public void start() {
-    monitorThread = new Thread(this);
+    monitorThread = new Thread(this, "ambari-hearbeat-monitor");
     monitorThread.start();
   }
 

ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java

@@ -62,6 +62,7 @@ import org.apache.ambari.server.agent.ExecutionCommand;
 import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.HostsMap;
 import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
+import org.apache.ambari.server.orm.entities.RequestEntity;
 import org.apache.ambari.server.serveraction.MockServerAction;
 import org.apache.ambari.server.serveraction.ServerActionExecutor;
 import org.apache.ambari.server.state.Cluster;
@@ -160,9 +161,9 @@ public class TestActionScheduler {
     when(db.getCommandsInProgressCount()).thenReturn(stages.size());
     when(db.getStagesInProgress()).thenReturn(stages);
 
-    Request request = mock(Request.class);
+    RequestEntity request = mock(RequestEntity.class);
     when(request.isExclusive()).thenReturn(false);
-    when(db.getRequest(anyLong())).thenReturn(request);
+    when(db.getRequestEntity(anyLong())).thenReturn(request);
 
     //Keep large number of attempts so that the task is not expired finally
     //Small action timeout to test rescheduling
@@ -252,9 +253,10 @@ public class TestActionScheduler {
     ActionDBAccessor db = mock(ActionDBAccessor.class);
     when(db.getCommandsInProgressCount()).thenReturn(stages.size());
     when(db.getStagesInProgress()).thenReturn(stages);
-    Request request = mock(Request.class);
+
+    RequestEntity request = mock(RequestEntity.class);
     when(request.isExclusive()).thenReturn(false);
-    when(db.getRequest(anyLong())).thenReturn(request);
+    when(db.getRequestEntity(anyLong())).thenReturn(request);
 
     doAnswer(new Answer() {
       @Override
@@ -330,9 +332,10 @@ public class TestActionScheduler {
 
     ActionDBAccessor db = mock(ActionDBAccessor.class);
 
-    Request request = mock(Request.class);
+    RequestEntity request = mock(RequestEntity.class);
     when(request.isExclusive()).thenReturn(false);
-    when(db.getRequest(anyLong())).thenReturn(request);
+    when(db.getRequestEntity(anyLong())).thenReturn(request);
+
     when(db.getCommandsInProgressCount()).thenReturn(stages.size());
     when(db.getStagesInProgress()).thenReturn(stages);
     doAnswer(new Answer() {
@@ -416,9 +419,10 @@ public class TestActionScheduler {
 
     ActionDBAccessor db = mock(ActionDBAccessor.class);
 
-    Request request = mock(Request.class);
+    RequestEntity request = mock(RequestEntity.class);
     when(request.isExclusive()).thenReturn(false);
-    when(db.getRequest(anyLong())).thenReturn(request);
+    when(db.getRequestEntity(anyLong())).thenReturn(request);
+
     when(db.getCommandsInProgressCount()).thenReturn(stages.size());
     when(db.getStagesInProgress()).thenReturn(stages);
 
@@ -549,9 +553,10 @@ public class TestActionScheduler {
 
     ActionDBAccessor db = mock(ActionDBAccessor.class);
 
-    Request request = mock(Request.class);
+    RequestEntity request = mock(RequestEntity.class);
     when(request.isExclusive()).thenReturn(false);
-    when(db.getRequest(anyLong())).thenReturn(request);
+    when(db.getRequestEntity(anyLong())).thenReturn(request);
+
     when(db.getCommandsInProgressCount()).thenReturn(stages.size());
     when(db.getStagesInProgress()).thenReturn(stages);
     doAnswer(new Answer() {
@@ -642,9 +647,10 @@ public class TestActionScheduler {
 
     ActionDBAccessor db = mock(ActionDBAccessor.class);
 
-    Request request = mock(Request.class);
+    RequestEntity request = mock(RequestEntity.class);
     when(request.isExclusive()).thenReturn(false);
-    when(db.getRequest(anyLong())).thenReturn(request);
+    when(db.getRequestEntity(anyLong())).thenReturn(request);
+
     when(db.getCommandsInProgressCount()).thenReturn(stages.size());
     when(db.getStagesInProgress()).thenReturn(stages);
     doAnswer(new Answer() {
@@ -727,9 +733,10 @@ public class TestActionScheduler {
 
     ActionDBAccessor db = mock(ActionDBAccessor.class);
 
-    Request request = mock(Request.class);
+    RequestEntity request = mock(RequestEntity.class);
     when(request.isExclusive()).thenReturn(false);
-    when(db.getRequest(anyLong())).thenReturn(request);
+    when(db.getRequestEntity(anyLong())).thenReturn(request);
+
     when(db.getCommandsInProgressCount()).thenReturn(stages.size());
     when(db.getStagesInProgress()).thenReturn(stages);
     doAnswer(new Answer() {
@@ -858,9 +865,10 @@ public class TestActionScheduler {
 
     ActionDBAccessor db = mock(ActionDBAccessor.class);
 
-    Request request = mock(Request.class);
+    RequestEntity request = mock(RequestEntity.class);
     when(request.isExclusive()).thenReturn(false);
-    when(db.getRequest(anyLong())).thenReturn(request);
+    when(db.getRequestEntity(anyLong())).thenReturn(request);
+
     when(db.getCommandsInProgressCount()).thenReturn(stages.size());
     when(db.getStagesInProgress()).thenReturn(stages);
 
@@ -949,9 +957,10 @@ public class TestActionScheduler {
 
     ActionDBAccessor db = mock(ActionDBAccessor.class);
 
-    Request request = mock(Request.class);
+    RequestEntity request = mock(RequestEntity.class);
     when(request.isExclusive()).thenReturn(false);
-    when(db.getRequest(anyLong())).thenReturn(request);
+    when(db.getRequestEntity(anyLong())).thenReturn(request);
+
     when(db.getCommandsInProgressCount()).thenReturn(stages.size());
     when(db.getStagesInProgress()).thenReturn(stages);
 
@@ -1025,9 +1034,10 @@ public class TestActionScheduler {
 
     ActionDBAccessor db = mock(ActionDBAccessor.class);
 
-    Request request = mock(Request.class);
+    RequestEntity request = mock(RequestEntity.class);
     when(request.isExclusive()).thenReturn(false);
-    when(db.getRequest(anyLong())).thenReturn(request);
+    when(db.getRequestEntity(anyLong())).thenReturn(request);
+
     when(db.getCommandsInProgressCount()).thenReturn(stages.size());
     when(db.getStagesInProgress()).thenReturn(stages);
 
@@ -1087,9 +1097,10 @@ public class TestActionScheduler {
 
     ActionDBAccessor db = mock(ActionDBAccessor.class);
 
-    Request request = mock(Request.class);
+    RequestEntity request = mock(RequestEntity.class);
     when(request.isExclusive()).thenReturn(false);
-    when(db.getRequest(anyLong())).thenReturn(request);
+    when(db.getRequestEntity(anyLong())).thenReturn(request);
+
     when(db.getCommandsInProgressCount()).thenReturn(stages.size());
     when(db.getStagesInProgress()).thenReturn(stages);
     doAnswer(new Answer() {
@@ -1268,9 +1279,10 @@ public class TestActionScheduler {
 
     ActionDBAccessor db = mock(ActionDBAccessor.class);
 
-    Request request = mock(Request.class);
+    RequestEntity request = mock(RequestEntity.class);
     when(request.isExclusive()).thenReturn(false);
-    when(db.getRequest(anyLong())).thenReturn(request);
+    when(db.getRequestEntity(anyLong())).thenReturn(request);
+
     when(db.getCommandsInProgressCount()).thenReturn(stages.size());
     when(db.getStagesInProgress()).thenReturn(stages);
     doAnswer(new Answer() {
@@ -1445,9 +1457,10 @@ public class TestActionScheduler {
 
     ActionDBAccessor db = mock(ActionDBAccessor.class);
 
-    Request request = mock(Request.class);
+    RequestEntity request = mock(RequestEntity.class);
     when(request.isExclusive()).thenReturn(false);
-    when(db.getRequest(anyLong())).thenReturn(request);
+    when(db.getRequestEntity(anyLong())).thenReturn(request);
+
     when(db.getCommandsInProgressCount()).thenReturn(stages.size());
     when(db.getStagesInProgress()).thenReturn(stages);
     doAnswer(new Answer() {
@@ -1673,9 +1686,9 @@ public class TestActionScheduler {
 
     ActionDBAccessor db = mock(ActionDBAccessorImpl.class);
 
-    Request request = mock(Request.class);
+    RequestEntity request = mock(RequestEntity.class);
     when(request.isExclusive()).thenReturn(false);
-    when(db.getRequest(anyLong())).thenReturn(request);
+    when(db.getRequestEntity(anyLong())).thenReturn(request);
 
     Stage s1 = StageUtils.getATestStage(requestId1, stageId, hostname, CLUSTER_HOST_INFO,
       "{\"host_param\":\"param_value\"}", "{\"stage_param\":\"param_value\"}");
@@ -1760,9 +1773,10 @@ public class TestActionScheduler {
 
     ActionDBAccessor db = mock(ActionDBAccessor.class);
 
-    Request request = mock(Request.class);
+    RequestEntity request = mock(RequestEntity.class);
     when(request.isExclusive()).thenReturn(false);
-    when(db.getRequest(anyLong())).thenReturn(request);
+    when(db.getRequestEntity(anyLong())).thenReturn(request);
+
     when(db.getCommandsInProgressCount()).thenReturn(stages.size());
     when(db.getStagesInProgress()).thenReturn(stages);
 
@@ -1841,9 +1855,10 @@ public class TestActionScheduler {
 
     ActionDBAccessor db = mock(ActionDBAccessor.class);
 
-    Request request = mock(Request.class);
+    RequestEntity request = mock(RequestEntity.class);
     when(request.isExclusive()).thenReturn(false);
-    when(db.getRequest(anyLong())).thenReturn(request);
+    when(db.getRequestEntity(anyLong())).thenReturn(request);
+
     when(db.getCommandsInProgressCount()).thenReturn(stages.size());
     when(db.getStagesInProgress()).thenReturn(stages);
     doAnswer(new Answer() {
@@ -1936,9 +1951,10 @@ public class TestActionScheduler {
 
     ActionDBAccessor db = mock(ActionDBAccessor.class);
 
-    Request request = mock(Request.class);
+    RequestEntity request = mock(RequestEntity.class);
     when(request.isExclusive()).thenReturn(false);
-    when(db.getRequest(anyLong())).thenReturn(request);
+    when(db.getRequestEntity(anyLong())).thenReturn(request);
+
     when(db.getCommandsInProgressCount()).thenReturn(stages.size());
     when(db.getStagesInProgress()).thenReturn(stages);
 
@@ -2164,16 +2180,16 @@ public class TestActionScheduler {
       }
     }).when(db).startRequest(anyLong());
 
-    Request request1 = mock(Request.class);
+    RequestEntity request1 = mock(RequestEntity.class);
     when(request1.isExclusive()).thenReturn(false);
-    Request request2 = mock(Request.class);
+    RequestEntity request2 = mock(RequestEntity.class);
     when(request2.isExclusive()).thenReturn(true);
-    Request request3 = mock(Request.class);
+    RequestEntity request3 = mock(RequestEntity.class);
     when(request3.isExclusive()).thenReturn(false);
 
-    when(db.getRequest(requestId1)).thenReturn(request1);
-    when(db.getRequest(requestId2)).thenReturn(request2);
-    when(db.getRequest(requestId3)).thenReturn(request3);
+    when(db.getRequestEntity(requestId1)).thenReturn(request1);
+    when(db.getRequestEntity(requestId2)).thenReturn(request2);
+    when(db.getRequestEntity(requestId3)).thenReturn(request3);
 
     Properties properties = new Properties();
     Configuration conf = new Configuration(properties);