AMBARI-9897. RU: query for UpgradeItem is taking too long on large upgrades (ncole)

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java

@@ -17,15 +17,19 @@
  */
 package org.apache.ambari.server.controller.internal;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
 
 import org.apache.ambari.server.actionmanager.HostRoleCommand;
 import org.apache.ambari.server.actionmanager.HostRoleStatus;
 import org.apache.ambari.server.actionmanager.Stage;
+import org.apache.ambari.server.orm.dao.HostRoleCommandStatusSummaryDTO;
 import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
 import org.apache.ambari.server.orm.entities.StageEntity;
 
@@ -227,6 +231,72 @@ public class CalculatedStatus {
     return calculateStatusCounts(hostRoleStatuses);
   }
 
+  /**
+   * Return counts of task statuses.
+   * @param stageDto  the map of stage-to-summary value objects
+   * @param stageIds  the stage ids to consider from the value objects
+   * @return the map of status to counts
+   */
+  public static Map<HostRoleStatus, Integer> calculateTaskStatusCounts(
+      Map<Long, HostRoleCommandStatusSummaryDTO> stageDto, Set<Long> stageIds) {
+
+    Map<HostRoleStatus, Integer> result = new HashMap<HostRoleStatus, Integer>();
+
+    for (Long stageId : stageIds) {
+      if (!stageDto.containsKey(stageId)) {
+        continue;
+      }
+
+      HostRoleCommandStatusSummaryDTO dto = stageDto.get(stageId);
+
+      for (Entry<HostRoleStatus, Integer> entry : dto.getCounts().entrySet()) {
+        if (!result.containsKey(entry.getKey())) {
+          result.put(entry.getKey(), 0);
+        }
+
+        Integer old = result.get(entry.getKey());
+        result.put(entry.getKey(), old + entry.getValue());
+      }
+
+    }
+
+    return result;
+  }
+
+  /**
+   * Calculates the overall status
+   * @param stageDto  the map of stage-to-summary value objects
+   * @param stageIds  the stage ids to consider from the value objects
+   * @return the calculated status
+   */
+  public static CalculatedStatus statusFromStageSummary(Map<Long, HostRoleCommandStatusSummaryDTO> stageDto,
+      Set<Long> stageIds) {
+
+    Collection<HostRoleStatus> stageStatuses = new HashSet<HostRoleStatus>();
+    Collection<HostRoleStatus> taskStatuses = new ArrayList<HostRoleStatus>();
+
+    for (Long stageId : stageIds) {
+      if (!stageDto.containsKey(stageId)) {
+        continue;
+      }
+
+      HostRoleCommandStatusSummaryDTO summary = stageDto.get(stageId);
+      HostRoleStatus stageStatus = calculateSummaryStatus(summary.getCounts(),
+          summary.getTaskTotal(), summary.isStageSkippable());
+
+      stageStatuses.add(stageStatus);
+
+      taskStatuses.addAll(summary.getTaskStatuses());
+    }
+
+    // calculate the overall status from the stage statuses
+    HostRoleStatus status = calculateSummaryStatus(calculateStatusCounts(stageStatuses), stageStatuses.size(), false);
+
+    double progressPercent = calculateProgressPercent(calculateStatusCounts(taskStatuses), taskStatuses.size());
+
+    return new CalculatedStatus(status, progressPercent);
+  }
+
   /**
    * Returns counts of tasks that are in various states.
    *

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java

@@ -17,6 +17,20 @@
  */
 package org.apache.ambari.server.controller.internal;
 
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.inject.Inject;
+import javax.inject.Provider;
+
 import org.apache.ambari.server.StaticallyInject;
 import org.apache.ambari.server.actionmanager.HostRoleStatus;
 import org.apache.ambari.server.controller.AmbariManagementController;
@@ -34,24 +48,13 @@ import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
 import org.apache.ambari.server.controller.utilities.PredicateBuilder;
 import org.apache.ambari.server.controller.utilities.PropertyHelper;
 import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
+import org.apache.ambari.server.orm.dao.HostRoleCommandStatusSummaryDTO;
 import org.apache.ambari.server.orm.dao.StageDAO;
 import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
 import org.apache.ambari.server.orm.entities.StageEntity;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 
-import javax.inject.Inject;
-import javax.inject.Provider;
-import java.util.Collection;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
 /**
  * ResourceProvider for Stage
  */
@@ -212,10 +215,18 @@ public class StageResourceProvider extends AbstractControllerResourceProvider im
     Set<Resource> results     = new LinkedHashSet<Resource>();
     Set<String>   propertyIds = getRequestPropertyIds(request, predicate);
 
+    // !!! poor mans cache.  toResource() shouldn't be calling the db
+    // every time, when the request id is likely the same for each stageEntity
+    Map<Long, Map<Long, HostRoleCommandStatusSummaryDTO>> cache =
+        new HashMap<Long, Map<Long, HostRoleCommandStatusSummaryDTO>>();
+
     List<StageEntity> entities = dao.findAll(request, predicate);
     for (StageEntity entity : entities) {
-      results.add(toResource(entity, propertyIds));
+      results.add(toResource(cache, entity, propertyIds));
     }
+
+    cache.clear();
+
     return results;
   }
 
@@ -301,8 +312,10 @@ public class StageResourceProvider extends AbstractControllerResourceProvider im
    *
    * @return the new resource
    */
-  private Resource toResource(StageEntity entity,
-                              Set<String> requestedIds) {
+  private Resource toResource(
+      Map<Long, Map<Long, HostRoleCommandStatusSummaryDTO>> cache,
+      StageEntity entity,
+      Set<String> requestedIds) {
 
     Resource resource = new ResourceImpl(Resource.Type.Stage);
 
@@ -317,6 +330,12 @@ public class StageResourceProvider extends AbstractControllerResourceProvider im
       }
     }
 
+    if (!cache.containsKey(entity.getRequestId())) {
+      cache.put(entity.getRequestId(), hostRoleCommandDAO.findAggregateCounts(entity.getRequestId()));
+    }
+
+    Map<Long, HostRoleCommandStatusSummaryDTO> summary = cache.get(entity.getRequestId());
+
     setResourceProperty(resource, STAGE_STAGE_ID, entity.getStageId(), requestedIds);
     setResourceProperty(resource, STAGE_REQUEST_ID, entity.getRequestId(), requestedIds);
     setResourceProperty(resource, STAGE_CONTEXT, entity.getRequestContext(), requestedIds);
@@ -325,20 +344,17 @@ public class StageResourceProvider extends AbstractControllerResourceProvider im
     setResourceProperty(resource, STAGE_HOST_PARAMS, entity.getHostParamsStage(), requestedIds);
     setResourceProperty(resource, STAGE_SKIPPABLE, entity.isSkippable(), requestedIds);
 
-    Collection<HostRoleCommandEntity> tasks = entity.getHostRoleCommands();
-
-    Long startTime = tasks.isEmpty() ? 0L : Long.MAX_VALUE;
-    Long endTime   = 0L;
-
-    for (HostRoleCommandEntity task : tasks) {
-      startTime = Math.min(task.getStartTime(), startTime);
-      endTime   = Math.max(task.getEndTime(), endTime);
+    Long startTime = Long.MAX_VALUE;
+    Long endTime = 0L;
+    if (summary.containsKey(entity.getStageId())) {
+      startTime = summary.get(entity.getStageId()).getStartTime();
+      endTime = summary.get(entity.getStageId()).getEndTime();
     }
 
     setResourceProperty(resource, STAGE_START_TIME, startTime, requestedIds);
     setResourceProperty(resource, STAGE_END_TIME, endTime, requestedIds);
 
-    CalculatedStatus status = CalculatedStatus.statusFromTaskEntities(tasks, entity.isSkippable());
+    CalculatedStatus status = CalculatedStatus.statusFromStageSummary(summary, Collections.singleton(entity.getStageId()));
 
     setResourceProperty(resource, STAGE_PROGRESS_PERCENT, status.getPercent(), requestedIds);
     setResourceProperty(resource, STAGE_STATUS, status.getStatus().toString(), requestedIds);

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeGroupResourceProvider.java

@@ -17,7 +17,6 @@
  */
 package org.apache.ambari.server.controller.internal;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -38,10 +37,10 @@ import org.apache.ambari.server.controller.spi.Resource;
 import org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException;
 import org.apache.ambari.server.controller.spi.SystemException;
 import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
+import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
+import org.apache.ambari.server.orm.dao.HostRoleCommandStatusSummaryDTO;
 import org.apache.ambari.server.orm.dao.StageDAO;
 import org.apache.ambari.server.orm.dao.UpgradeDAO;
-import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
-import org.apache.ambari.server.orm.entities.StageEntity;
 import org.apache.ambari.server.orm.entities.UpgradeEntity;
 import org.apache.ambari.server.orm.entities.UpgradeGroupEntity;
 import org.apache.ambari.server.orm.entities.UpgradeItemEntity;
@@ -82,6 +81,9 @@ public class UpgradeGroupResourceProvider extends AbstractControllerResourceProv
   @Inject
   private static StageDAO stageDAO = null;
 
+  @Inject
+  private static HostRoleCommandDAO s_hostRoleCommandDao;
+
   static {
     // properties
     PROPERTY_IDS.add(UPGRADE_REQUEST_ID);
@@ -139,7 +141,9 @@ public class UpgradeGroupResourceProvider extends AbstractControllerResourceProv
       List<UpgradeGroupEntity> groups = upgrade.getUpgradeGroups();
       if (null != groups) {
 
-        for (UpgradeGroupEntity group : upgrade.getUpgradeGroups()) {
+        Map<Long, HostRoleCommandStatusSummaryDTO> map = s_hostRoleCommandDao.findAggregateCounts(requestId);
+
+        for (UpgradeGroupEntity group : groups) {
           Resource r = toResource(upgrade, group, requestPropertyIds);
 
           Set<Long> stageIds = new HashSet<Long>();
@@ -147,7 +151,7 @@ public class UpgradeGroupResourceProvider extends AbstractControllerResourceProv
             stageIds.add(itemEntity.getStageId());
           }
 
-          aggregate(r, requestId, stageIds, requestPropertyIds);
+          aggregate(map, r, requestId, stageIds, requestPropertyIds);
 
           results.add(r);
         }
@@ -196,15 +200,19 @@ public class UpgradeGroupResourceProvider extends AbstractControllerResourceProv
    * @param stageIds      the set of resources ids of the stages
    * @param requestedIds  the ids for the request
    */
-  private void aggregate(Resource upgradeGroup, Long requestId, Set<Long> stageIds, Set<String> requestedIds) {
-    List<StageEntity> stages = stageDAO.findByStageIds(requestId, stageIds);
+  private void aggregate(Map<Long, HostRoleCommandStatusSummaryDTO> smap,
+      Resource upgradeGroup, Long requestId, Set<Long> stageIds, Set<String> requestedIds) {
+
+    int size = 0;
 
-    List<HostRoleCommandEntity> tasks = new ArrayList<HostRoleCommandEntity>();
-    for (StageEntity stage : stages) {
-      tasks.addAll(stage.getHostRoleCommands());
+    Map<HostRoleStatus, Integer> counts = CalculatedStatus.calculateTaskStatusCounts(smap, stageIds);
+
+    CalculatedStatus stageStatus = CalculatedStatus.statusFromStageSummary(smap, stageIds);
+
+    for (Integer i : counts.values()) {
+      size += i.intValue();
     }
 
-    Map<HostRoleStatus, Integer> counts = CalculatedStatus.calculateTaskEntityStatusCounts(tasks);
     Integer inProgress = 0;
     Integer completed = 0;
 
@@ -215,13 +223,11 @@ public class UpgradeGroupResourceProvider extends AbstractControllerResourceProv
         inProgress += entry.getValue();
       }
     }
-    setResourceProperty(upgradeGroup, UPGRADE_GROUP_TOTAL_TASKS, tasks.size(), requestedIds);
+    setResourceProperty(upgradeGroup, UPGRADE_GROUP_TOTAL_TASKS, size, requestedIds);
     setResourceProperty(upgradeGroup, UPGRADE_GROUP_IN_PROGRESS_TASKS, inProgress, requestedIds);
     setResourceProperty(upgradeGroup, UPGRADE_GROUP_COMPLETED_TASKS, completed, requestedIds);
 
-    CalculatedStatus status = CalculatedStatus.statusFromStageEntities(stages);
-
-    setResourceProperty(upgradeGroup, UPGRADE_GROUP_STATUS, status.getStatus(), requestedIds);
-    setResourceProperty(upgradeGroup, UPGRADE_GROUP_PROGRESS_PERCENT, status.getPercent(), requestedIds);
+    setResourceProperty(upgradeGroup, UPGRADE_GROUP_STATUS, stageStatus.getStatus(), requestedIds);
+    setResourceProperty(upgradeGroup, UPGRADE_GROUP_PROGRESS_PERCENT, stageStatus.getPercent(), requestedIds);
   }
 }

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java

@@ -28,7 +28,6 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -60,9 +59,13 @@ import org.apache.ambari.server.controller.spi.Resource;
 import org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException;
 import org.apache.ambari.server.controller.spi.SystemException;
 import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
+import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
+import org.apache.ambari.server.orm.dao.HostRoleCommandStatusSummaryDTO;
 import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
+import org.apache.ambari.server.orm.dao.RequestDAO;
 import org.apache.ambari.server.orm.dao.UpgradeDAO;
 import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
+import org.apache.ambari.server.orm.entities.RequestEntity;
 import org.apache.ambari.server.orm.entities.UpgradeEntity;
 import org.apache.ambari.server.orm.entities.UpgradeGroupEntity;
 import org.apache.ambari.server.orm.entities.UpgradeItemEntity;
@@ -109,6 +112,21 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
   protected static final String UPGRADE_REQUEST_STATUS = "Upgrade/request_status";
   protected static final String UPGRADE_ABORT_REASON = "Upgrade/abort_reason";
 
+
+  /*
+   * Lifted from RequestResourceProvider
+   */
+  private static final String REQUEST_CONTEXT_ID = "Upgrade/request_context";
+  private static final String REQUEST_TYPE_ID = "Upgrade/type";
+  private static final String REQUEST_CREATE_TIME_ID = "Upgrade/create_time";
+  private static final String REQUEST_START_TIME_ID = "Upgrade/start_time";
+  private static final String REQUEST_END_TIME_ID = "Upgrade/end_time";
+  private static final String REQUEST_EXCLUSIVE_ID = "Upgrade/exclusive";
+
+  private static final String REQUEST_PROGRESS_PERCENT_ID = "Upgrade/progress_percent";
+  private static final String REQUEST_STATUS_PROPERTY_ID = "Upgrade/request_status";
+
+
   private static final Set<String> PK_PROPERTY_IDS = new HashSet<String>(
       Arrays.asList(UPGRADE_REQUEST_ID, UPGRADE_CLUSTER_NAME));
   private static final Set<String> PROPERTY_IDS = new HashSet<String>();
@@ -145,6 +163,12 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
   @Inject
   private static Provider<AmbariCustomCommandExecutionHelper> s_commandExecutionHelper;
 
+  @Inject
+  private static RequestDAO s_requestDAO = null;
+
+  @Inject
+  private static HostRoleCommandDAO s_hostRoleCommandDAO = null;
+
   /**
    * Used to generated the correct tasks and stages during an upgrade.
    */
@@ -154,8 +178,6 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
   @Inject
   private static Configuration s_configuration;
 
-  private static Map<String, String> REQUEST_PROPERTY_MAP = new HashMap<String, String>();
-
   static {
     // properties
     PROPERTY_IDS.add(UPGRADE_CLUSTER_NAME);
@@ -166,11 +188,14 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
     PROPERTY_IDS.add(UPGRADE_TO_VERSION);
     PROPERTY_IDS.add(UPGRADE_DIRECTION);
 
-    // !!! boo
-    for (String requestPropertyId : RequestResourceProvider.PROPERTY_IDS) {
-      REQUEST_PROPERTY_MAP.put(requestPropertyId, requestPropertyId.replace("Requests/", "Upgrade/"));
-    }
-    PROPERTY_IDS.addAll(REQUEST_PROPERTY_MAP.values());
+    PROPERTY_IDS.add(REQUEST_CONTEXT_ID);
+    PROPERTY_IDS.add(REQUEST_CREATE_TIME_ID);
+    PROPERTY_IDS.add(REQUEST_END_TIME_ID);
+    PROPERTY_IDS.add(REQUEST_EXCLUSIVE_ID);
+    PROPERTY_IDS.add(REQUEST_PROGRESS_PERCENT_ID);
+    PROPERTY_IDS.add(REQUEST_START_TIME_ID);
+    PROPERTY_IDS.add(REQUEST_STATUS_PROPERTY_ID);
+    PROPERTY_IDS.add(REQUEST_TYPE_ID);
 
     // keys
     KEY_PROPERTY_IDS.put(Resource.Type.Upgrade, UPGRADE_REQUEST_ID);
@@ -274,15 +299,22 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
         Resource r = toResource(entity, clusterName, requestPropertyIds);
         results.add(r);
 
-        // !!! not terribly efficient, but that's ok in this case.  The handful-per-year
-        // an upgrade is done won't kill performance.
-        Resource r1 = s_upgradeHelper.getRequestResource(clusterName,
-            entity.getRequestId());
-        for (Entry<String, String> entry : REQUEST_PROPERTY_MAP.entrySet()) {
-          Object o = r1.getPropertyValue(entry.getKey());
+        RequestEntity rentity = s_requestDAO.findByPK(entity.getRequestId());
 
-          setResourceProperty(r, entry.getValue(), o, requestPropertyIds);
-        }
+        setResourceProperty(r, REQUEST_CONTEXT_ID, rentity.getRequestContext(), requestPropertyIds);
+        setResourceProperty(r, REQUEST_TYPE_ID, rentity.getRequestType(), requestPropertyIds);
+        setResourceProperty(r, REQUEST_CREATE_TIME_ID, rentity.getCreateTime(), requestPropertyIds);
+        setResourceProperty(r, REQUEST_START_TIME_ID, rentity.getStartTime(), requestPropertyIds);
+        setResourceProperty(r, REQUEST_END_TIME_ID, rentity.getEndTime(), requestPropertyIds);
+        setResourceProperty(r, REQUEST_EXCLUSIVE_ID, rentity.isExclusive(), requestPropertyIds);
+
+        Map<Long, HostRoleCommandStatusSummaryDTO> summary = s_hostRoleCommandDAO.findAggregateCounts(entity.getRequestId());
+
+        CalculatedStatus calc = CalculatedStatus.statusFromStageSummary(
+            summary, summary.keySet());
+
+        setResourceProperty(r, REQUEST_STATUS_PROPERTY_ID, calc.getStatus(), requestPropertyIds);
+        setResourceProperty(r, REQUEST_PROGRESS_PERCENT_ID, calc.getPercent(), requestPropertyIds);
       }
     }
 

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java

@@ -46,6 +46,26 @@ import com.google.inject.persist.Transactional;
 @Singleton
 public class HostRoleCommandDAO {
 
+  private static final String SUMMARY_DTO = String.format(
+    "SELECT NEW %s(" +
+      "MIN(hrc.startTime), " +
+      "MAX(hrc.endTime), " +
+      "MIN(hrc.stage.skippable), " +
+      "hrc.stageId, " +
+      "SUM(CASE WHEN hrc.status = :aborted THEN 1 ELSE 0 END), " +
+      "SUM(CASE WHEN hrc.status = :completed THEN 1 ELSE 0 END), " +
+      "SUM(CASE WHEN hrc.status = :failed THEN 1 ELSE 0 END), " +
+      "SUM(CASE WHEN hrc.status = :holding THEN 1 ELSE 0 END), " +
+      "SUM(CASE WHEN hrc.status = :holding_failed THEN 1 ELSE 0 END), " +
+      "SUM(CASE WHEN hrc.status = :holding_timedout THEN 1 ELSE 0 END), " +
+      "SUM(CASE WHEN hrc.status = :in_progress THEN 1 ELSE 0 END), " +
+      "SUM(CASE WHEN hrc.status = :pending THEN 1 ELSE 0 END), " +
+      "SUM(CASE WHEN hrc.status = :queued THEN 1 ELSE 0 END), " +
+      "SUM(CASE WHEN hrc.status = :timedout THEN 1 ELSE 0 END)" +
+      ") FROM HostRoleCommandEntity hrc " +
+      " GROUP BY hrc.requestId, hrc.stageId HAVING hrc.requestId = :requestId",
+      HostRoleCommandStatusSummaryDTO.class.getName());
+
   @Inject
   Provider<EntityManager> entityManagerProvider;
   @Inject
@@ -349,4 +369,40 @@ public class HostRoleCommandDAO {
   public void removeByPK(int taskId) {
     remove(findByPK(taskId));
   }
+
+
+  /**
+   * Finds the counts of tasks for a request and groups them by stage id.
+   * This allows for very efficient loading when there are a huge number of stages
+   * and tasks to iterate (for example, during a Rolling Upgrade).
+   * @param requestId the request id
+   * @return the map of stage-to-summary objects
+   */
+  @RequiresSession
+  public Map<Long, HostRoleCommandStatusSummaryDTO> findAggregateCounts(Long requestId) {
+
+    TypedQuery<HostRoleCommandStatusSummaryDTO> query = entityManagerProvider.get().createQuery(
+        SUMMARY_DTO, HostRoleCommandStatusSummaryDTO.class);
+
+    query.setParameter("requestId", requestId);
+    query.setParameter("aborted", HostRoleStatus.ABORTED);
+    query.setParameter("completed", HostRoleStatus.COMPLETED);
+    query.setParameter("failed", HostRoleStatus.FAILED);
+    query.setParameter("holding", HostRoleStatus.HOLDING);
+    query.setParameter("holding_failed", HostRoleStatus.HOLDING_FAILED);
+    query.setParameter("holding_timedout", HostRoleStatus.HOLDING_TIMEDOUT);
+    query.setParameter("in_progress", HostRoleStatus.IN_PROGRESS);
+    query.setParameter("pending", HostRoleStatus.PENDING);
+    query.setParameter("queued", HostRoleStatus.QUEUED);
+    query.setParameter("timedout", HostRoleStatus.TIMEDOUT);
+
+    Map<Long, HostRoleCommandStatusSummaryDTO> map = new HashMap<Long, HostRoleCommandStatusSummaryDTO>();
+
+    for (HostRoleCommandStatusSummaryDTO dto : daoUtils.selectList(query)) {
+      map.put(dto.getStageId(), dto);
+    }
+
+    return map;
+  }
+
 }

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandStatusSummaryDTO.java

@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.orm.dao;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
+
+/**
+ * Summary for a single stage with summary counts for the tasks for that stage.
+ */
+public class HostRoleCommandStatusSummaryDTO {
+
+  private Long m_stageId = Long.valueOf(0L);
+  private Long m_minTime = Long.valueOf(0L);
+  private Long m_maxTime = Long.valueOf(Long.MAX_VALUE);
+  private boolean m_skippable = false;
+  private Map<HostRoleStatus, Integer> m_counts = new HashMap<HostRoleStatus, Integer>();
+  private List<HostRoleStatus> m_tasksStatus = new ArrayList<HostRoleStatus>();
+
+  /**
+   * Constructor invoked by JPA.  See {{@link HostRoleCommandDAO#findAggregateCounts(Long)}}
+   */
+  public HostRoleCommandStatusSummaryDTO(
+      Number skippable,
+      Number minStartTime,
+      Number maxEndTime,
+      Number stageId,
+      Number aborted,
+      Number completed,
+      Number failed,
+      Number holding,
+      Number holdingFailed,
+      Number holdingTimedout,
+      Number inProgress,
+      Number pending,
+      Number queued,
+      Number timedout) {
+
+    m_stageId = Long.valueOf(null == stageId ? 0L : stageId.longValue());
+    if (null != skippable) {
+      m_skippable = (1 == skippable.intValue());
+    }
+    if (null != minStartTime) {
+      m_minTime = Long.valueOf(minStartTime.longValue());
+    }
+    if (null != maxEndTime) {
+      m_maxTime = Long.valueOf(maxEndTime.longValue());
+    }
+
+    put(HostRoleStatus.ABORTED, aborted);
+    put(HostRoleStatus.COMPLETED, completed);
+    put(HostRoleStatus.FAILED, failed);
+    put(HostRoleStatus.HOLDING, holding);
+    put(HostRoleStatus.HOLDING_FAILED, holdingFailed);
+    put(HostRoleStatus.HOLDING_TIMEDOUT, holdingTimedout);
+    put(HostRoleStatus.IN_PROGRESS, inProgress);
+    put(HostRoleStatus.PENDING, pending);
+    put(HostRoleStatus.QUEUED, queued);
+    put(HostRoleStatus.TIMEDOUT, timedout);
+
+  }
+
+  @SuppressWarnings("boxing")
+  private void put(HostRoleStatus status, Number number) {
+    if (null != number) {
+      m_counts.put(status, number.intValue());
+      for (int i = 0; i < number.intValue(); i++) {
+        m_tasksStatus.add(status);
+      }
+    } else {
+      m_counts.put(status, 0);
+    }
+  }
+
+  /**
+   * @return the stage id for this summary
+   */
+  Long getStageId() {
+    return m_stageId;
+  }
+
+  /**
+   * @return the task result counts, by status
+   */
+  public Map<HostRoleStatus, Integer> getCounts() {
+    return m_counts;
+  }
+
+  /**
+   * @return the list of tasks status, expanded to cover all tasks for the stage
+   */
+  public List<HostRoleStatus> getTaskStatuses() {
+    return m_tasksStatus;
+  }
+
+  /**
+   * @return the total number of tasks for the stage
+   */
+  public int getTaskTotal() {
+    return m_tasksStatus.size();
+  }
+
+  /**
+   * @return {@code true} if the stage is skippable
+   */
+  public boolean isStageSkippable() {
+    return m_skippable;
+  }
+
+  /**
+   * @return the start time, minimum, for the tasks
+   */
+  public Long getStartTime() {
+    return m_minTime;
+  }
+
+  /**
+   * @return the end time, maximum, for the tasks
+   */
+  public Long getEndTime() {
+    return m_maxTime;
+  }
+
+}

ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java

@@ -28,7 +28,6 @@ import java.util.regex.Pattern;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
-import org.apache.ambari.server.controller.internal.RequestResourceProvider;
 import org.apache.ambari.server.controller.internal.StageResourceProvider;
 import org.apache.ambari.server.controller.predicate.AndPredicate;
 import org.apache.ambari.server.controller.spi.ClusterController;
@@ -482,40 +481,6 @@ public class UpgradeHelper {
     return response.getResources();
   }
 
-  /**
-   * Gets a Request resource to aggreate with an Upgrade
-   * @param clusterName the cluster name
-   * @param requestId the request id
-   * @return the resource for the Request
-   * @throws UnsupportedPropertyException
-   * @throws NoSuchResourceException
-   * @throws NoSuchParentResourceException
-   * @throws SystemException
-   */
-  // !!! FIXME this feels very wrong
-  public Resource getRequestResource(String clusterName, Long requestId)
-      throws UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException, SystemException {
-
-    ClusterController clusterController = ClusterControllerHelper.getClusterController();
-
-    Request request = PropertyHelper.getReadRequest();
-
-    Predicate predicate = new PredicateBuilder().property(RequestResourceProvider.REQUEST_CLUSTER_NAME_PROPERTY_ID).equals(clusterName).and()
-        // !!! RequestResourceProvider is expecting a string, not a Long for the requestId
-        .property(RequestResourceProvider.REQUEST_ID_PROPERTY_ID).equals(requestId.toString()).toPredicate();
-
-    QueryResponse response = clusterController.getResources(Resource.Type.Request,
-        request, predicate);
-
-    Set<Resource> resources = response.getResources();
-    if (1 != resources.size()) {
-      throw new SystemException(String.format(
-          "Cannot uniquely identify the request resource for %s", requestId));
-    }
-
-    return resources.iterator().next();
-  }
-
   /**
    * Helper to set service and component display names on the context
    * @param context   the context to update

ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ColocatedGrouping.java

@@ -22,6 +22,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -236,7 +237,7 @@ public class ColocatedGrouping extends Grouping {
      * @param wrappers  the list of stage wrappers
      */
     private void formatFirstBatch(UpgradeContext ctx, ManualTask task, List<StageWrapper> wrappers) {
-      List<String> compNames = new ArrayList<String>();
+      Set<String> names = new LinkedHashSet<String>();
       Map<String, Set<String>> compLocations = new HashMap<String, Set<String>>();
 
       for (StageWrapper sw : wrappers) {
@@ -251,7 +252,7 @@ public class ColocatedGrouping extends Grouping {
               compLocations.get(host).add(tw.getComponent());
             }
 
-            compNames.add(ctx.getComponentDisplay(
+            names.add(ctx.getComponentDisplay(
                 tw.getService(), tw.getComponent()));
           }
         }
@@ -261,9 +262,11 @@ public class ColocatedGrouping extends Grouping {
       if (task.message.contains("{{components}}")) {
         StringBuilder sb = new StringBuilder();
 
+        List<String> compNames = new ArrayList<String>(names);
+
         if (compNames.size() == 1) {
           sb.append(compNames.get(0));
-        } else if (compNames.size() > 1) {
+        } else if (names.size() > 1) {
           String last = compNames.remove(compNames.size() - 1);
           sb.append(StringUtils.join(compNames, ", "));
           sb.append(" and ").append(last);

ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StageResourceProviderTest.java

@@ -17,11 +17,25 @@
  */
 package org.apache.ambari.server.controller.internal;
 
-import com.google.inject.Binder;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Module;
-import com.google.inject.util.Modules;
+import static org.easymock.EasyMock.anyLong;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.createStrictMock;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.fail;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.ambari.server.actionmanager.ActionManager;
 import org.apache.ambari.server.actionmanager.HostRoleStatus;
 import org.apache.ambari.server.controller.AmbariManagementController;
@@ -33,32 +47,23 @@ import org.apache.ambari.server.controller.utilities.PredicateBuilder;
 import org.apache.ambari.server.controller.utilities.PropertyHelper;
 import org.apache.ambari.server.metadata.ActionMetadata;
 import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
+import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
+import org.apache.ambari.server.orm.dao.HostRoleCommandStatusSummaryDTO;
 import org.apache.ambari.server.orm.dao.StageDAO;
 import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
 import org.apache.ambari.server.orm.entities.StageEntity;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
+import org.easymock.EasyMock;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.easymock.EasyMock.anyLong;
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.createStrictMock;
-import static org.easymock.EasyMock.eq;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.verify;
-import static org.junit.Assert.*;
+import com.google.inject.Binder;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import com.google.inject.util.Modules;
 
 public class StageResourceProviderTest {
 
@@ -67,12 +72,23 @@ public class StageResourceProviderTest {
   private Cluster cluster = null;
   private AmbariManagementController managementController = null;
   private Injector injector;
+  private HostRoleCommandDAO hrcDao = null;
 
   @Before
   public void before() {
     dao = createStrictMock(StageDAO.class);
     clusters = createStrictMock(Clusters.class);
     cluster = createStrictMock(Cluster.class);
+    hrcDao = createStrictMock(HostRoleCommandDAO.class);
+
+    expect(hrcDao.findAggregateCounts(EasyMock.anyObject(Long.class))).andReturn(
+        new HashMap<Long, HostRoleCommandStatusSummaryDTO>() {{
+          put(0L, new HostRoleCommandStatusSummaryDTO(
+              0, 1000L, 2500L, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0));
+        }}).anyTimes();
+
+    replay(hrcDao);
+
     managementController = createNiceMock(AmbariManagementController.class);
 
     // create an injector which will inject the mocks
@@ -243,6 +259,7 @@ public class StageResourceProviderTest {
       binder.bind(StageDAO.class).toInstance(dao);
       binder.bind(Clusters.class).toInstance(clusters);
       binder.bind(Cluster.class).toInstance(cluster);
+      binder.bind(HostRoleCommandDAO.class).toInstance(hrcDao);
       binder.bind(AmbariManagementController.class).toInstance(managementController);
       binder.bind(ActionMetadata.class);
     }

ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java

@@ -32,6 +32,7 @@ import java.util.Set;
 
 import org.apache.ambari.server.actionmanager.ActionManager;
 import org.apache.ambari.server.actionmanager.HostRoleCommand;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
 import org.apache.ambari.server.actionmanager.RequestStatus;
 import org.apache.ambari.server.actionmanager.Stage;
 import org.apache.ambari.server.api.resources.UpgradeResourceDefinition;
@@ -46,9 +47,11 @@ import org.apache.ambari.server.controller.utilities.PropertyHelper;
 import org.apache.ambari.server.orm.GuiceJpaInitializer;
 import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
 import org.apache.ambari.server.orm.OrmTestHelper;
+import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
 import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
 import org.apache.ambari.server.orm.dao.StageDAO;
 import org.apache.ambari.server.orm.dao.UpgradeDAO;
+import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
 import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
 import org.apache.ambari.server.orm.entities.StageEntity;
 import org.apache.ambari.server.orm.entities.UpgradeEntity;
@@ -539,6 +542,54 @@ public class UpgradeResourceProviderTest {
   }
 
 
+  @Test
+  public void testPercents() throws Exception {
+    org.apache.ambari.server.controller.spi.RequestStatus status = testCreateResources();
+
+    Set<Resource> createdResources = status.getAssociatedResources();
+    assertEquals(1, createdResources.size());
+    Resource res = createdResources.iterator().next();
+    Long id = (Long) res.getPropertyValue("Upgrade/request_id");
+    assertNotNull(id);
+    assertEquals(Long.valueOf(1), id);
+
+    StageDAO stageDao = injector.getInstance(StageDAO.class);
+    HostRoleCommandDAO hrcDao = injector.getInstance(HostRoleCommandDAO.class);
+
+    List<StageEntity> stages = stageDao.findByRequestId(id);
+    List<HostRoleCommandEntity> tasks = hrcDao.findByRequest(id);
+
+    Set<Long> stageIds = new HashSet<Long>();
+    for (StageEntity se : stages) {
+      stageIds.add(se.getStageId());
+    }
+
+    CalculatedStatus calc = null;
+    int i = 0;
+    for (HostRoleCommandEntity hrce : tasks) {
+      hrce.setStatus(HostRoleStatus.IN_PROGRESS);
+      hrcDao.merge(hrce);
+      calc = CalculatedStatus.statusFromStageSummary(hrcDao.findAggregateCounts(id), stageIds);
+      assertEquals(((i++) + 1) * 4.375d, calc.getPercent(), 0.01d);
+      assertEquals(HostRoleStatus.IN_PROGRESS, calc.getStatus());
+    }
+
+    i = 0;
+    for (HostRoleCommandEntity hrce : tasks) {
+      hrce.setStatus(HostRoleStatus.COMPLETED);
+      hrcDao.merge(hrce);
+      calc = CalculatedStatus.statusFromStageSummary(hrcDao.findAggregateCounts(id), stageIds);
+      assertEquals(35 + (((i++) + 1) * 8.125), calc.getPercent(), 0.01d);
+      if (i < 8) {
+        assertEquals(HostRoleStatus.IN_PROGRESS, calc.getStatus());
+      }
+    }
+
+    calc = CalculatedStatus.statusFromStageSummary(hrcDao.findAggregateCounts(id), stageIds);
+    assertEquals(HostRoleStatus.COMPLETED, calc.getStatus());
+    assertEquals(100d, calc.getPercent(), 0.01d);
+  }
+
   /**
    * @param amc
    * @return the provider