AMBARI-3135. Out of memory issues with Request API on large cluster. (Myroslav Papirkovskyy via swagle)

ambari-project/pom.xml

@@ -198,6 +198,11 @@
         <artifactId>postgresql</artifactId>
         <version>9.1-901.jdbc4</version>
       </dependency>
+      <dependency>
+        <groupId>com.google.guava</groupId>
+        <artifactId>guava</artifactId>
+        <version>14.0.1</version>
+      </dependency>
       <dependency>
         <groupId>org.mockito</groupId>
         <artifactId>mockito-core</artifactId>

ambari-server/pom.xml

@@ -149,6 +149,26 @@
           <!--</execution>-->
         <!--</executions>-->
       <!--</plugin>-->
+      <plugin>
+        <!--Static -->
+        <artifactId>maven-antrun-plugin</artifactId>
+        <executions>
+          <execution>
+            <phase>process-classes</phase>
+            <configuration>
+              <tasks>
+                <java classname="org.eclipse.persistence.tools.weaving.jpa.StaticWeave"
+                      classpathref="maven.runtime.classpath" fork="true">
+                  <arg line="-loglevel FINE -persistenceinfo src/main/resources target/classes target/classes"/>
+                </java>
+              </tasks>
+            </configuration>
+            <goals>
+              <goal>run</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>org.codehaus.mojo</groupId>
         <artifactId>rpm-maven-plugin</artifactId>
@@ -741,6 +761,11 @@
 	  <artifactId>httpclient</artifactId>
 	  <version>4.2.5</version>
     </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>14.0.1</version>
+    </dependency>
   </dependencies>
   <!--<reporting>
         <plugins>

ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java

@@ -18,7 +18,12 @@
 package org.apache.ambari.server.actionmanager;
 
 import java.util.*;
+import java.util.concurrent.TimeUnit;
 
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.name.Named;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.Role;
 import org.apache.ambari.server.agent.CommandReport;
@@ -67,13 +72,20 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
   @Inject
   private Clusters clusters;
 
+  private Cache<Long, HostRoleCommand> hostRoleCommandCache;
+  private long cacheLimit; //may be exceeded to store tasks from one request
+
   private final long requestId;
 
   @Inject
-  public ActionDBAccessorImpl(Injector injector) {
+  public ActionDBAccessorImpl(Injector injector, @Named("executionCommandCacheSize") long cacheLimit) {
     injector.injectMembers(this);
     requestId = stageDAO.getLastRequestId();
 
+    this.cacheLimit = cacheLimit;
+    hostRoleCommandCache = CacheBuilder.newBuilder().
+        expireAfterAccess(5, TimeUnit.MINUTES).
+        build();
 
   }
 
@@ -105,7 +117,7 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
     List<HostRoleCommandEntity> commands =
         hostRoleCommandDAO.findByRequest(requestId);
     for (HostRoleCommandEntity command : commands) {
-      if(command.getStatus() == HostRoleStatus.QUEUED ||
+      if (command.getStatus() == HostRoleStatus.QUEUED ||
           command.getStatus() == HostRoleStatus.IN_PROGRESS ||
           command.getStatus() == HostRoleStatus.PENDING) {
         command.setStatus(HostRoleStatus.ABORTED);
@@ -125,7 +137,7 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
   @Override
   @Transactional
   public void timeoutHostRole(String host, long requestId, long stageId,
-      Role role) {
+                              Role role) {
     List<HostRoleCommandEntity> commands =
         hostRoleCommandDAO.findByHostRole(host, requestId, stageId, role);
     for (HostRoleCommandEntity command : commands) {
@@ -179,7 +191,7 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
         HostEntity hostEntity = hostDAO.findByName(hostRoleCommandEntity.getHostName());
         if (hostEntity == null) {
           LOG.error("Host {} doesn't exists in database" + hostRoleCommandEntity.getHostName());
-          throw new RuntimeException("Host '"+hostRoleCommandEntity.getHostName()+"' doesn't exists in database");
+          throw new RuntimeException("Host '" + hostRoleCommandEntity.getHostName() + "' doesn't exists in database");
         }
         hostRoleCommandEntity.setHost(hostEntity);
         hostRoleCommandDAO.create(hostRoleCommandEntity);
@@ -208,7 +220,7 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
   @Override
   @Transactional
   public void updateHostRoleState(String hostname, long requestId,
-      long stageId, String role, CommandReport report) {
+                                  long stageId, String role, CommandReport report) {
     if (LOG.isDebugEnabled()) {
       LOG.debug("Update HostRoleState: "
           + "HostName " + hostname + " requestId " + requestId + " stageId "
@@ -260,10 +272,14 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
   @Override
   public List<HostRoleCommand> getRequestTasks(long requestId) {
     List<HostRoleCommand> tasks = new ArrayList<HostRoleCommand>();
-    for (HostRoleCommandEntity hostRoleCommandEntity : hostRoleCommandDAO.findByRequest(requestId)) {
-      tasks.add(hostRoleCommandFactory.createExisting(hostRoleCommandEntity));
-    }
-    return tasks;
+    return getTasks(
+        hostRoleCommandDAO.findTaskIdsByRequest(requestId)
+    );
+
+//    for (HostRoleCommandEntity hostRoleCommandEntity : hostRoleCommandDAO.findByRequest(requestId)) {
+//      tasks.add(hostRoleCommandFactory.createExisting(hostRoleCommandEntity));
+//    }
+//    return tasks;
   }
 
   @Override
@@ -271,24 +287,31 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
     if (requestIds.isEmpty()) {
       return Collections.emptyList();
     }
-    List<HostRoleCommand> tasks = new ArrayList<HostRoleCommand>();
-    for (HostRoleCommandEntity hostRoleCommandEntity : hostRoleCommandDAO.findByRequestIds(requestIds)) {
-      tasks.add(hostRoleCommandFactory.createExisting(hostRoleCommandEntity));
-    }
-    return tasks;
+
+    return getTasks(
+        hostRoleCommandDAO.findTaskIdsByRequestIds(requestIds)
+    );
+
+//    List<HostRoleCommand> tasks = new ArrayList<HostRoleCommand>();
+//    for (HostRoleCommandEntity hostRoleCommandEntity : hostRoleCommandDAO.findByRequestIds(requestIds)) {
+//      tasks.add(hostRoleCommandFactory.createExisting(hostRoleCommandEntity));
+//    }
+//    return tasks;
   }
 
   @Override
   public List<HostRoleCommand> getTasksByRequestAndTaskIds(Collection<Long> requestIds, Collection<Long> taskIds) {
     if (!requestIds.isEmpty() && !taskIds.isEmpty()) {
-      List<HostRoleCommand> tasks = new ArrayList<HostRoleCommand>();
-      for (HostRoleCommandEntity hostRoleCommandEntity : hostRoleCommandDAO.findByRequestAndTaskIds(requestIds, taskIds)) {
-        tasks.add(hostRoleCommandFactory.createExisting(hostRoleCommandEntity));
-      }
-      return tasks;
-    }else if (requestIds.isEmpty()) {
+      return getTasks(hostRoleCommandDAO.findTaskIdsByRequestAndTaskIds(requestIds, taskIds));
+
+//      List<HostRoleCommand> tasks = new ArrayList<HostRoleCommand>();
+//      for (HostRoleCommandEntity hostRoleCommandEntity : hostRoleCommandDAO.findByRequestAndTaskIds(requestIds, taskIds)) {
+//        tasks.add(hostRoleCommandFactory.createExisting(hostRoleCommandEntity));
+//      }
+//      return tasks;
+    } else if (requestIds.isEmpty()) {
       return getTasks(taskIds);
-    }else if (taskIds.isEmpty()) {
+    } else if (taskIds.isEmpty()) {
       return getAllTasksByRequestIds(requestIds);
     } else {
       return Collections.emptyList();
@@ -300,10 +323,36 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
     if (taskIds.isEmpty()) {
       return Collections.emptyList();
     }
+
     List<HostRoleCommand> commands = new ArrayList<HostRoleCommand>();
-    for (HostRoleCommandEntity commandEntity : hostRoleCommandDAO.findByPKs(taskIds)) {
-      commands.add(hostRoleCommandFactory.createExisting(commandEntity));
+
+    Map<Long, HostRoleCommand> cached = hostRoleCommandCache.getAllPresent(taskIds);
+    commands.addAll(cached.values());
+
+    List<Long> absent = new ArrayList<Long>();
+    absent.addAll(taskIds);
+    absent.removeAll(cached.keySet());
+
+    if (!absent.isEmpty()) {
+      boolean allowStore = hostRoleCommandCache.size() <= cacheLimit;
+//      LOG.info("Cache size {}, enable = {}", hostRoleCommandCache.size(), allowStore);
+
+      for (HostRoleCommandEntity commandEntity : hostRoleCommandDAO.findByPKs(absent)) {
+        HostRoleCommand hostRoleCommand = hostRoleCommandFactory.createExisting(commandEntity);
+        commands.add(hostRoleCommand);
+        if (allowStore) {
+          switch (hostRoleCommand.getStatus()) {
+            case ABORTED:
+            case COMPLETED:
+            case TIMEDOUT:
+            case FAILED:
+              hostRoleCommandCache.put(hostRoleCommand.getTaskId(), hostRoleCommand);
+              break;
+          }
+        }
+      }
     }
+
     return commands;
   }
 
@@ -322,7 +371,7 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
   }
 
   public HostRoleCommand getTask(long taskId) {
-    HostRoleCommandEntity commandEntity = hostRoleCommandDAO.findByPK((int)taskId);
+    HostRoleCommandEntity commandEntity = hostRoleCommandDAO.findByPK((int) taskId);
     if (commandEntity == null) {
       return null;
     }
@@ -335,17 +384,17 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
     boolean checkAllTasks = false;
     Set<HostRoleStatus> statuses = new HashSet<HostRoleStatus>();
     if (status == RequestStatus.IN_PROGRESS) {
-      statuses.addAll( Arrays.asList(HostRoleStatus.PENDING,
+      statuses.addAll(Arrays.asList(HostRoleStatus.PENDING,
           HostRoleStatus.IN_PROGRESS, HostRoleStatus.QUEUED));
     } else if (status == RequestStatus.COMPLETED) {
       match = false;
       checkAllTasks = true;
-      statuses.addAll( Arrays.asList(HostRoleStatus.PENDING,
+      statuses.addAll(Arrays.asList(HostRoleStatus.PENDING,
           HostRoleStatus.IN_PROGRESS, HostRoleStatus.QUEUED,
           HostRoleStatus.ABORTED, HostRoleStatus.FAILED,
           HostRoleStatus.TIMEDOUT));
     } else if (status == RequestStatus.FAILED) {
-      statuses.addAll( Arrays.asList(HostRoleStatus.ABORTED,
+      statuses.addAll(Arrays.asList(HostRoleStatus.ABORTED,
           HostRoleStatus.FAILED, HostRoleStatus.TIMEDOUT));
     }
     return hostRoleCommandDAO.getRequestsByTaskStatus(statuses, match, checkAllTasks);

ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommand.java

@@ -19,6 +19,7 @@ package org.apache.ambari.server.actionmanager;
 
 import org.apache.ambari.server.Role;
 import org.apache.ambari.server.RoleCommand;
+import org.apache.ambari.server.orm.dao.ExecutionCommandDAO;
 import org.apache.ambari.server.orm.entities.ExecutionCommandEntity;
 import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
 import org.apache.ambari.server.state.ServiceComponentHostEvent;
@@ -26,8 +27,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.inject.Injector;
-import com.google.inject.assistedinject.Assisted;
-import com.google.inject.assistedinject.AssistedInject;
 
 /**
  * This class encapsulates the information for an task on a host for a
@@ -56,6 +55,8 @@ public class HostRoleCommand {
 
   private ExecutionCommandWrapper executionCommandWrapper;
 
+  private ExecutionCommandDAO executionCommandDAO;
+
   public HostRoleCommand(String host, Role role,
       ServiceComponentHostEvent event, RoleCommand command) {
     this.hostName = host;
@@ -64,8 +65,7 @@ public class HostRoleCommand {
     this.roleCommand = command;
   }
 
-  @AssistedInject
-  public HostRoleCommand(@Assisted HostRoleCommandEntity hostRoleCommandEntity, Injector injector) {
+  public HostRoleCommand(HostRoleCommandEntity hostRoleCommandEntity, Injector injector) {
     taskId = hostRoleCommandEntity.getTaskId();
     stageId = hostRoleCommandEntity.getStage().getStageId();
     requestId = hostRoleCommandEntity.getStage().getRequestId();
@@ -80,10 +80,13 @@ public class HostRoleCommand {
     attemptCount = hostRoleCommandEntity.getAttemptCount();
     roleCommand = hostRoleCommandEntity.getRoleCommand();
     event = new ServiceComponentHostEventWrapper(hostRoleCommandEntity.getEvent());
-    executionCommandWrapper = new ExecutionCommandWrapper(new String(
-        hostRoleCommandEntity
-            .getExecutionCommand().getCommand()
-    ));
+    //make use of lazy loading
+
+    executionCommandDAO = injector.getInstance(ExecutionCommandDAO.class);
+//    executionCommandWrapper = new ExecutionCommandWrapper(new String(
+//        hostRoleCommandEntity
+//            .getExecutionCommand().getCommand()
+//    ));
   }
 
   HostRoleCommandEntity constructNewPersistenceEntity() {
@@ -198,6 +201,16 @@ public class HostRoleCommand {
   }
 
   public ExecutionCommandWrapper getExecutionCommandWrapper() {
+    if (taskId != -1 && executionCommandWrapper == null) {
+      ExecutionCommandEntity commandEntity = executionCommandDAO.findByPK(taskId);
+      if (commandEntity == null) {
+        throw new RuntimeException("Invalid DB state, broken one-to-one relation for taskId=" + taskId);
+      }
+      executionCommandWrapper = new ExecutionCommandWrapper(new String(
+          commandEntity.getCommand()
+      ));
+    }
+
     return executionCommandWrapper;
   }
 

ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommandFactoryImpl.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.actionmanager;
+
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.Singleton;
+import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
+
+@Singleton
+public class HostRoleCommandFactoryImpl implements HostRoleCommandFactory {
+  private Injector injector;
+
+  @Inject
+  public HostRoleCommandFactoryImpl(Injector injector) {
+    this.injector = injector;
+  }
+
+  @Override
+  public HostRoleCommand createExisting(HostRoleCommandEntity hostRoleCommandEntity) {
+    return new HostRoleCommand(hostRoleCommandEntity, injector);
+  }
+}

ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java

@@ -129,6 +129,9 @@ public class Configuration {
   public static final String ADMIN_ROLE_NAME_KEY =
       "authorization.adminRoleName";
 
+  public static final String SERVER_EC_CACHE_SIZE = "server.ecCacheSize";
+  private static final long SERVER_EC_CACHE_SIZE_DEFAULT = 10000L;
+
   public static final String SERVER_PERSISTENCE_TYPE_KEY = "server.persistence.type";
   public static final String SERVER_JDBC_USER_NAME_KEY = "server.jdbc.user.name";
   public static final String SERVER_JDBC_USER_PASSWD_KEY = "server.jdbc.user.passwd";
@@ -805,4 +808,18 @@ public class Configuration {
     return ambariPropertiesMap;
   }
 
+  public long getExecutionCommandsCacheSize() {
+    String stringValue = properties.getProperty(SERVER_EC_CACHE_SIZE);
+    long value = SERVER_EC_CACHE_SIZE_DEFAULT;
+    if (stringValue != null) {
+      try {
+        value = Long.getLong(stringValue);
+      } catch (NumberFormatException ignored) {
+      }
+
+    }
+
+    return value;
+  }
+
 }

ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java

@@ -79,6 +79,12 @@ public class ControllerModule extends AbstractModule {
     bind(ActionDBAccessor.class).to(ActionDBAccessorImpl.class);
     bindConstant().annotatedWith(Names.named("schedulerSleeptime")).to(10000L);
     bindConstant().annotatedWith(Names.named("actionTimeout")).to(600000L);
+
+    //ExecutionCommands cache size
+
+    bindConstant().annotatedWith(Names.named("executionCommandCacheSize")).
+        to(configuration.getExecutionCommandsCacheSize());
+
     bind(AmbariManagementController.class)
         .to(AmbariManagementControllerImpl.class);
     bind(AbstractRootServiceResponseFactory.class).to(RootServiceResponseFactory.class);
@@ -158,7 +164,7 @@ public class ControllerModule extends AbstractModule {
     install(new FactoryModuleBuilder().implement(
         Config.class, ConfigImpl.class).build(ConfigFactory.class));
     install(new FactoryModuleBuilder().build(StageFactory.class));
-    install(new FactoryModuleBuilder().build(HostRoleCommandFactory.class));
+    bind(HostRoleCommandFactory.class).to(HostRoleCommandFactoryImpl.class);
   }
 
 }

ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/PropertyHelper.java

@@ -191,6 +191,9 @@ public class PropertyHelper {
    * @return true if the given property id contains any replacement arguments
    */
   public static boolean containsArguments(String propertyId) {
+    if (!propertyId.contains("$")) {
+      return false;
+    }
     Matcher matcher = CHECK_FOR_METRIC_ARGUMENTS_REGEX.matcher(propertyId);
     return matcher.find();
   }

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ClusterDAO.java

@@ -24,6 +24,7 @@ import javax.persistence.EntityManager;
 import javax.persistence.NoResultException;
 import javax.persistence.TypedQuery;
 
+import com.google.inject.Singleton;
 import org.apache.ambari.server.orm.entities.ClusterConfigEntity;
 import org.apache.ambari.server.orm.entities.ClusterEntity;
 
@@ -31,6 +32,7 @@ import com.google.inject.Inject;
 import com.google.inject.Provider;
 import com.google.inject.persist.Transactional;
 
+@Singleton
 public class ClusterDAO {
 
   @Inject

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ClusterServiceDAO.java

@@ -20,6 +20,7 @@ package org.apache.ambari.server.orm.dao;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.orm.entities.ClusterServiceEntity;
 import org.apache.ambari.server.orm.entities.ClusterServiceEntityPK;
@@ -28,6 +29,7 @@ import javax.persistence.EntityManager;
 import javax.persistence.NoResultException;
 import javax.persistence.TypedQuery;
 
+@Singleton
 public class ClusterServiceDAO {
   @Inject
   Provider<EntityManager> entityManagerProvider;

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ClusterStateDAO.java

@@ -20,11 +20,13 @@ package org.apache.ambari.server.orm.dao;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.orm.entities.ClusterStateEntity;
 
 import javax.persistence.EntityManager;
 
+@Singleton
 public class ClusterStateDAO {
   @Inject
   Provider<EntityManager> entityManagerProvider;

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ComponentConfigMappingDAO.java

@@ -24,6 +24,7 @@ import java.util.List;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 
 import javax.persistence.EntityManager;
@@ -31,6 +32,7 @@ import javax.persistence.TypedQuery;
 
 import org.apache.ambari.server.orm.entities.ComponentConfigMappingEntity;
 
+@Singleton
 public class ComponentConfigMappingDAO {
 
   @Inject

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ExecutionCommandDAO.java

@@ -20,11 +20,13 @@ package org.apache.ambari.server.orm.dao;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.orm.entities.ExecutionCommandEntity;
 
 import javax.persistence.EntityManager;
 
+@Singleton
 public class ExecutionCommandDAO {
 
   @Inject

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentConfigMappingDAO.java

@@ -24,6 +24,7 @@ import java.util.List;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 
 import javax.persistence.EntityManager;
@@ -31,6 +32,7 @@ import javax.persistence.TypedQuery;
 
 import org.apache.ambari.server.orm.entities.HostComponentConfigMappingEntity;
 
+@Singleton
 public class HostComponentConfigMappingDAO {
 
   @Inject

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentDesiredConfigMappingDAO.java

@@ -24,6 +24,7 @@ import java.util.List;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 
 import javax.persistence.EntityManager;
@@ -31,6 +32,7 @@ import javax.persistence.TypedQuery;
 
 import org.apache.ambari.server.orm.entities.HostComponentDesiredConfigMappingEntity;
 
+@Singleton
 public class HostComponentDesiredConfigMappingDAO {
 
   @Inject

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentDesiredStateDAO.java

@@ -20,12 +20,14 @@ package org.apache.ambari.server.orm.dao;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntity;
 import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntityPK;
 
 import javax.persistence.EntityManager;
 
+@Singleton
 public class HostComponentDesiredStateDAO {
   @Inject
   Provider<EntityManager> entityManagerProvider;

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostComponentStateDAO.java

@@ -20,12 +20,14 @@ package org.apache.ambari.server.orm.dao;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.orm.entities.HostComponentStateEntity;
 import org.apache.ambari.server.orm.entities.HostComponentStateEntityPK;
 
 import javax.persistence.EntityManager;
 
+@Singleton
 public class HostComponentStateDAO {
   @Inject
   Provider<EntityManager> entityManagerProvider;

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostConfigMappingDAO.java

@@ -22,6 +22,7 @@ import java.util.*;
 import javax.persistence.EntityManager;
 import javax.persistence.TypedQuery;
 
+import com.google.inject.Singleton;
 import org.apache.ambari.server.orm.entities.HostConfigMappingEntity;
 
 import com.google.inject.Inject;
@@ -31,6 +32,7 @@ import com.google.inject.persist.Transactional;
 /**
  * Used for host configuration mapping operations.
  */
+@Singleton
 public class HostConfigMappingDAO {
   @Inject
   Provider<EntityManager> entityManagerProvider;

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java

@@ -20,6 +20,7 @@ package org.apache.ambari.server.orm.dao;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.orm.entities.HostEntity;
 import org.apache.ambari.server.orm.entities.StageEntity;
@@ -30,6 +31,7 @@ import javax.persistence.TypedQuery;
 import java.util.Collections;
 import java.util.List;
 
+@Singleton
 public class HostDAO {
 
   @Inject

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java

@@ -20,6 +20,7 @@ package org.apache.ambari.server.orm.dao;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.Role;
 import org.apache.ambari.server.actionmanager.HostRoleStatus;
@@ -36,6 +37,7 @@ import javax.persistence.TypedQuery;
 import java.util.Collection;
 import java.util.List;
 
+@Singleton
 public class HostRoleCommandDAO {
 
   @Inject
@@ -68,6 +70,15 @@ public class HostRoleCommandDAO {
     return daoUtils.selectList(query, requestIds);
   }
 
+  @Transactional
+  public List<Long> findTaskIdsByRequestIds(Collection<Long> requestIds) {
+    TypedQuery<Long> query = entityManagerProvider.get().createQuery(
+        "SELECT task.taskId FROM HostRoleCommandEntity task " +
+            "WHERE task.requestId IN ?1 " +
+            "ORDER BY task.taskId", Long.class);
+    return daoUtils.selectList(query, requestIds);
+  }
+
   @Transactional
   public List<HostRoleCommandEntity> findByRequestAndTaskIds(Collection<Long> requestIds, Collection<Long> taskIds) {
     TypedQuery<HostRoleCommandEntity> query = entityManagerProvider.get().createQuery(
@@ -78,6 +89,16 @@ public class HostRoleCommandDAO {
     return daoUtils.selectList(query, requestIds, taskIds);
   }
 
+  @Transactional
+  public List<Long> findTaskIdsByRequestAndTaskIds(Collection<Long> requestIds, Collection<Long> taskIds) {
+    TypedQuery<Long> query = entityManagerProvider.get().createQuery(
+        "SELECT DISTINCT task.taskId FROM HostRoleCommandEntity task " +
+            "WHERE task.requestId IN ?1 AND task.taskId IN ?2 " +
+            "ORDER BY task.taskId", Long.class
+    );
+    return daoUtils.selectList(query, requestIds, taskIds);
+  }
+
   @Transactional
   public List<HostRoleCommandEntity> findSortedCommandsByStageAndHost(StageEntity stageEntity, HostEntity hostEntity) {
     TypedQuery<HostRoleCommandEntity> query = entityManagerProvider.get().createQuery("SELECT hostRoleCommand " +
@@ -127,6 +148,14 @@ public class HostRoleCommandDAO {
         "WHERE command.requestId=?1 ORDER BY command.taskId", HostRoleCommandEntity.class);
     return daoUtils.selectList(query, requestId);
   }
+  
+  @Transactional
+  public List<Long> findTaskIdsByRequest(long requestId) {
+    TypedQuery<Long> query = entityManagerProvider.get().createQuery("SELECT command.taskId " +
+        "FROM HostRoleCommandEntity command " +
+        "WHERE command.requestId=?1 ORDER BY command.taskId", Long.class);
+    return daoUtils.selectList(query, requestId);
+  }
 
   @Transactional
   public void create(HostRoleCommandEntity stageEntity) {

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostStateDAO.java

@@ -20,11 +20,13 @@ package org.apache.ambari.server.orm.dao;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.orm.entities.HostStateEntity;
 
 import javax.persistence.EntityManager;
 
+@Singleton
 public class HostStateDAO {
   @Inject
   Provider<EntityManager> entityManagerProvider;

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KeyValueDAO.java

@@ -20,6 +20,7 @@ package org.apache.ambari.server.orm.dao;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.orm.entities.KeyValueEntity;
 
@@ -27,6 +28,7 @@ import javax.persistence.EntityManager;
 import javax.persistence.TypedQuery;
 import java.util.Collection;
 
+@Singleton
 public class KeyValueDAO {
 
   @Inject

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/MetainfoDAO.java

@@ -20,6 +20,7 @@ package org.apache.ambari.server.orm.dao;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.orm.entities.MetainfoEntity;
 
@@ -27,6 +28,7 @@ import javax.persistence.EntityManager;
 import javax.persistence.TypedQuery;
 import java.util.Collection;
 
+@Singleton
 public class MetainfoDAO {
 
   @Inject

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/RoleDAO.java

@@ -19,11 +19,13 @@ package org.apache.ambari.server.orm.dao;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.orm.entities.RoleEntity;
 
 import javax.persistence.EntityManager;
 
+@Singleton
 public class RoleDAO {
 
   @Inject

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/RoleSuccessCriteriaDAO.java

@@ -20,12 +20,14 @@ package org.apache.ambari.server.orm.dao;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.orm.entities.RoleSuccessCriteriaEntity;
 import org.apache.ambari.server.orm.entities.RoleSuccessCriteriaEntityPK;
 
 import javax.persistence.EntityManager;
 
+@Singleton
 public class RoleSuccessCriteriaDAO {
 
   @Inject

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ServiceComponentDesiredStateDAO.java

@@ -20,12 +20,14 @@ package org.apache.ambari.server.orm.dao;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntityPK;
 import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity;
 
 import javax.persistence.EntityManager;
 
+@Singleton
 public class ServiceComponentDesiredStateDAO {
   @Inject
   Provider<EntityManager> entityManagerProvider;

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ServiceConfigMappingDAO.java

@@ -19,6 +19,7 @@ package org.apache.ambari.server.orm.dao;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.orm.entities.ServiceConfigMappingEntity;
 
@@ -28,6 +29,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 
+@Singleton
 public class ServiceConfigMappingDAO {
   @Inject
   Provider<EntityManager> entityManagerProvider;

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ServiceDesiredStateDAO.java

@@ -20,12 +20,14 @@ package org.apache.ambari.server.orm.dao;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.orm.entities.ServiceDesiredStateEntity;
 import org.apache.ambari.server.orm.entities.ServiceDesiredStateEntityPK;
 
 import javax.persistence.EntityManager;
 
+@Singleton
 public class ServiceDesiredStateDAO {
   @Inject
   Provider<EntityManager> entityManagerProvider;

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/StageDAO.java

@@ -20,6 +20,7 @@ package org.apache.ambari.server.orm.dao;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.actionmanager.HostRoleStatus;
 import org.apache.ambari.server.orm.entities.StageEntity;
@@ -32,6 +33,7 @@ import java.util.List;
 import java.util.Collection;
 import java.util.Map;
 
+@Singleton
 public class StageDAO {
 
   @Inject

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/UserDAO.java

@@ -19,6 +19,7 @@ package org.apache.ambari.server.orm.dao;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.Singleton;
 import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.orm.entities.UserEntity;
 
@@ -27,6 +28,7 @@ import javax.persistence.NoResultException;
 import javax.persistence.TypedQuery;
 import java.util.List;
 
+@Singleton
 public class UserDAO {
 
   @Inject

ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity.java

@@ -30,7 +30,6 @@ import static org.apache.commons.lang.StringUtils.defaultString;
 
 @Table(name = "host_role_command")
 @Entity
-@Cacheable(false)
 @TableGenerator(name = "host_role_command_id_generator",
     table = "ambari_sequences", pkColumnName = "sequence_name", valueColumnName = "value"
     , pkColumnValue = "host_role_command_id_seq"
@@ -100,7 +99,7 @@ public class HostRoleCommandEntity {
   @Enumerated(EnumType.STRING)
   private RoleCommand roleCommand;
 
-  @OneToOne(mappedBy = "hostRoleCommand", cascade = CascadeType.REMOVE)
+  @OneToOne(mappedBy = "hostRoleCommand", cascade = CascadeType.REMOVE, fetch = FetchType.LAZY)
   private ExecutionCommandEntity executionCommand;
 
   @ManyToOne(cascade = {CascadeType.MERGE})

ambari-server/src/main/resources/META-INF/persistence.xml

@@ -42,8 +42,9 @@
     <properties>
       <!--<property name="javax.persistence.jdbc.url" value="jdbc:postgresql://localhost/ambari" />-->
       <!--<property name="javax.persistence.jdbc.driver" value="org.postgresql.Driver" />-->
-      <property name="eclipselink.cache.size.default" value="3000" />
+      <property name="eclipselink.cache.size.default" value="10000" />
       <property name="eclipselink.jdbc.batch-writing" value="JDBC"/>
+      <property name="eclipselink.weaving" value="static" />
     </properties>
   </persistence-unit>