AMBARI-13064 - Provide Summary Of Skipped Failures During Upgrade (jonathanhurley)

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java

@@ -35,7 +35,6 @@ import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import com.google.gson.Gson;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.Role;
 import org.apache.ambari.server.RoleCommand;
@@ -66,7 +65,6 @@ import org.apache.ambari.server.controller.spi.SystemException;
 import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
 import org.apache.ambari.server.controller.utilities.PredicateBuilder;
 import org.apache.ambari.server.controller.utilities.PropertyHelper;
-import org.apache.ambari.server.orm.dao.HostDAO;
 import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
 import org.apache.ambari.server.orm.dao.HostRoleCommandStatusSummaryDTO;
 import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
@@ -104,6 +102,7 @@ import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.gson.Gson;
 import com.google.inject.Inject;
 import com.google.inject.Provider;
 
@@ -204,9 +203,6 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
   @Inject
   private static HostRoleCommandDAO s_hostRoleCommandDAO = null;
 
-  @Inject
-  private static HostDAO s_hostDAO = null;
-
   /**
    * Used to generated the correct tasks and stages during an upgrade.
    */

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java

@@ -312,6 +312,35 @@ public class HostRoleCommandDAO {
     return daoUtils.selectAll(entityManagerProvider.get(), HostRoleCommandEntity.class);
   }
 
+  /**
+   * Finds all the {@link HostRoleCommandEntity}s for the given request that are
+   * between the specified stage IDs and have the specified status.
+   *
+   * @param requestId
+   *          the request ID
+   * @param status
+   *          the command status to query for (not {@code null}).
+   * @param minStageId
+   *          the lowest stage ID to requests tasks for.
+   * @param maxStageId
+   *          the highest stage ID to request tasks for.
+   * @return the tasks that satisfy the specified parameters.
+   */
+  @RequiresSession
+  public List<HostRoleCommandEntity> findByStatusBetweenStages(long requestId,
+      HostRoleStatus status, long minStageId, long maxStageId) {
+
+    TypedQuery<HostRoleCommandEntity> query = entityManagerProvider.get().createNamedQuery(
+        "HostRoleCommandEntity.findByStatusBetweenStages", HostRoleCommandEntity.class);
+
+    query.setParameter("requestId", requestId);
+    query.setParameter("status", status);
+    query.setParameter("minStageId", minStageId);
+    query.setParameter("maxStageId", maxStageId);
+
+    return daoUtils.selectList(query);
+  }
+
   /**
    * Gets requests that have tasks in any of the specified statuses.
    *

ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity.java

@@ -59,7 +59,8 @@ import org.apache.commons.lang.ArrayUtils;
     @NamedQuery(name = "HostRoleCommandEntity.findByCommandStatuses", query = "SELECT command FROM HostRoleCommandEntity command WHERE command.status IN :statuses ORDER BY command.requestId, command.stageId"),
     @NamedQuery(name = "HostRoleCommandEntity.findByHostId", query = "SELECT command FROM HostRoleCommandEntity command WHERE command.hostId=:hostId"),
     @NamedQuery(name = "HostRoleCommandEntity.findByHostRole", query = "SELECT command FROM HostRoleCommandEntity command WHERE command.hostEntity.hostName=:hostName AND command.requestId=:requestId AND command.stageId=:stageId AND command.role=:role ORDER BY command.taskId"),
-    @NamedQuery(name = "HostRoleCommandEntity.findByHostRoleNullHost", query = "SELECT command FROM HostRoleCommandEntity command WHERE command.hostEntity IS NULL AND command.requestId=:requestId AND command.stageId=:stageId AND command.role=:role")
+    @NamedQuery(name = "HostRoleCommandEntity.findByHostRoleNullHost", query = "SELECT command FROM HostRoleCommandEntity command WHERE command.hostEntity IS NULL AND command.requestId=:requestId AND command.stageId=:stageId AND command.role=:role"),
+    @NamedQuery(name = "HostRoleCommandEntity.findByStatusBetweenStages", query = "SELECT command FROM HostRoleCommandEntity command WHERE command.requestId = :requestId AND command.stageId >= :minStageId AND command.stageId <= :maxStageId AND command.status = :status")
 })
 public class HostRoleCommandEntity {
 
@@ -492,4 +493,20 @@ public class HostRoleCommandEntity {
   public void setTopologyLogicalTaskEntity(TopologyLogicalTaskEntity topologyLogicalTaskEntity) {
     this.topologyLogicalTaskEntity = topologyLogicalTaskEntity;
   }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public String toString() {
+    StringBuilder buffer = new StringBuilder("HostRoleCommandEntity{ ");
+    buffer.append("taskId").append(taskId);
+    buffer.append(", stageId=").append(stageId);
+    buffer.append(", requestId=").append(requestId);
+    buffer.append(", role=").append(role);
+    buffer.append(", roleCommand=").append(roleCommand);
+    buffer.append(", exitcode=").append(exitcode);
+    buffer.append("}");
+    return buffer.toString();
+  }
 }

ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/AutoSkipFailedSummaryAction.java

@@ -0,0 +1,189 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.serveraction.upgrades;
+
+import java.text.MessageFormat;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.actionmanager.HostRoleCommand;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.actionmanager.ServiceComponentHostEventWrapper;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
+import org.apache.ambari.server.orm.dao.UpgradeDAO;
+import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
+import org.apache.ambari.server.orm.entities.UpgradeGroupEntity;
+import org.apache.ambari.server.orm.entities.UpgradeItemEntity;
+import org.apache.ambari.server.serveraction.AbstractServerAction;
+import org.apache.ambari.server.state.ServiceComponentHostEvent;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gson.Gson;
+import com.google.inject.Inject;
+
+/**
+ * The {@link AutoSkipFailedSummaryAction} is used to check if any
+ * {@link HostRoleCommand}s were skipped automatically after they failed during
+ * an upgrade. This will be automatically marked as
+ * {@link HostRoleStatus#COMPLETED} if there are no skipped failures. Otherwise
+ * it will be placed into {@link HostRoleStatus#HOLDING}.
+ */
+public class AutoSkipFailedSummaryAction extends AbstractServerAction {
+
+  /**
+   * Logger.
+   */
+  private static final Logger LOG = LoggerFactory.getLogger(AutoSkipFailedSummaryAction.class);
+
+  /**
+   * The standard output template message.
+   */
+  private static final String FAILURE_STD_OUT_TEMPLATE = "There were {0} skipped failure(s) that must be addressed before you can proceed. Please resolve each failure before continuing with the upgrade.";
+
+  /**
+   * ...
+   */
+  private static final String MIDDLE_ELLIPSIZE_MARKER = "\n\u2026\n";
+
+  /**
+   * Used to lookup the {@link UpgradeGroupEntity}.
+   */
+  @Inject
+  private UpgradeDAO m_upgradeDAO;
+
+  /**
+   * Used to lookup the tasks that need to be checked for
+   * {@link HostRoleStatus#SKIPPED_FAILED}.
+   */
+  @Inject
+  private HostRoleCommandDAO m_hostRoleCommandDAO;
+
+  /**
+   * Used for writing structured out.
+   */
+  @Inject
+  private Gson m_gson;
+
+  /**
+   * A mapping of host -> Map<key,info> for each failure.
+   */
+  private Map<String, Map<String, Object>> m_structuredFailures = new HashMap<>();
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext)
+      throws AmbariException, InterruptedException {
+
+    HostRoleCommand hostRoleCommand = getHostRoleCommand();
+    long requestId = hostRoleCommand.getRequestId();
+    long stageId = hostRoleCommand.getStageId();
+
+    // use the host role command to get to the parent upgrade group
+    UpgradeItemEntity upgradeItem = m_upgradeDAO.findUpgradeItemByRequestAndStage(requestId,stageId);
+    UpgradeGroupEntity upgradeGroup = upgradeItem.getGroupEntity();
+
+    // find all of the stages in this group
+    long upgradeGroupId = upgradeGroup.getId();
+    UpgradeGroupEntity upgradeGroupEntity = m_upgradeDAO.findUpgradeGroup(upgradeGroupId);
+    List<UpgradeItemEntity> groupUpgradeItems = upgradeGroupEntity.getItems();
+    TreeSet<Long> stageIds = new TreeSet<>();
+    for (UpgradeItemEntity groupUpgradeItem : groupUpgradeItems) {
+      stageIds.add(groupUpgradeItem.getStageId());
+    }
+
+    // for every stage, find all tasks that have been SKIPPED_FAILED - we use a
+    // bit of trickery here since within any given request, the stage ID are
+    // always sequential. This allows us to make a simple query instead of some
+    // overly complex IN or NESTED SELECT query
+    long minStageId = stageIds.first();
+    long maxStageId = stageIds.last();
+
+    List<HostRoleCommandEntity> skippedTasks = m_hostRoleCommandDAO.findByStatusBetweenStages(
+        hostRoleCommand.getRequestId(),
+        HostRoleStatus.SKIPPED_FAILED, minStageId, maxStageId);
+
+    if (skippedTasks.isEmpty()) {
+      return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+          "There were no skipped failures", null);
+    }
+
+    StringBuilder buffer = new StringBuilder("The following steps failed and were automatically skipped:\n");
+
+    for (HostRoleCommandEntity skippedTask : skippedTasks) {
+      try{
+        ServiceComponentHostEventWrapper eventWrapper = new ServiceComponentHostEventWrapper(
+            skippedTask.getEvent());
+
+        ServiceComponentHostEvent event = eventWrapper.getEvent();
+
+        String hostName = skippedTask.getHostName();
+        if(null != hostName){
+          Map<String, Object> failures = m_structuredFailures.get(hostName);
+          if( null == failures ){
+            failures = new HashMap<>();
+            m_structuredFailures.put(hostName, failures);
+          }
+
+          failures.put("id", skippedTask.getTaskId());
+          failures.put("exit_code", skippedTask.getExitcode());
+          failures.put("output_log", skippedTask.getOutputLog());
+          failures.put("error_log", skippedTask.getErrorLog());
+
+          String stdOut = StringUtils.abbreviateMiddle(new String(skippedTask.getStdOut()),
+              MIDDLE_ELLIPSIZE_MARKER, 1000);
+
+          String stderr = StringUtils.abbreviateMiddle(new String(skippedTask.getStdError()),
+              MIDDLE_ELLIPSIZE_MARKER, 1000);
+
+          failures.put("stdout", stdOut);
+          failures.put("stderr", stderr);
+        }
+
+        buffer.append(event.getServiceComponentName());
+        if (null != event.getHostName()) {
+          buffer.append(" on ");
+          buffer.append(event.getHostName());
+        }
+
+        buffer.append(": ");
+        buffer.append(skippedTask.getCommandDetail());
+        buffer.append("\n");
+      } catch (Exception exception) {
+        LOG.warn("Unable to extract failure information for {}", skippedTask);
+        buffer.append(": ");
+        buffer.append(skippedTask);
+      }
+    }
+
+    String structuredOutput = m_gson.toJson(m_structuredFailures);
+    String standardOutput = MessageFormat.format(FAILURE_STD_OUT_TEMPLATE, skippedTasks.size());
+    String standardError = buffer.toString();
+
+    return createCommandReport(0, HostRoleStatus.HOLDING, structuredOutput, standardOutput,
+        standardError);
+  }
+}

ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java

@@ -439,9 +439,21 @@ public class UpgradeHelper {
      * List of stages for the group
      */
     public List<StageWrapper> items = new ArrayList<StageWrapper>();
-  }
-
 
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public String toString() {
+      StringBuilder buffer = new StringBuilder("UpgradeGroupHolder{ ");
+      buffer.append("name").append(name);
+      buffer.append(", title=").append(title);
+      buffer.append(", allowRetry=").append(allowRetry);
+      buffer.append(", skippable=").append(skippable);
+      buffer.append("}");
+      return buffer.toString();
+    }
+  }
 
   /**
    * Gets a set of Stages resources to aggregate an UpgradeItem with Stage.

ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java

@@ -94,6 +94,7 @@ public class UpgradePack {
       if (null == group.intendedDirection || direction == group.intendedDirection) {
         checked.add(group);
       }
+
     }
 
     return checked;

ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java

@@ -31,7 +31,6 @@ import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
-import javax.xml.bind.annotation.XmlTransient;
 import javax.xml.bind.annotation.XmlType;
 
 import org.apache.ambari.server.stack.HostsType;
@@ -59,12 +58,9 @@ public class ClusterGrouping extends Grouping {
   @XmlElement(name="execute-stage")
   public List<ExecuteStage> executionStages;
 
-  @XmlTransient
-  private ClusterBuilder m_builder = new ClusterBuilder();
-
   @Override
   public ClusterBuilder getBuilder() {
-    return m_builder;
+    return new ClusterBuilder(this);
   }
 
 
@@ -100,24 +96,39 @@ public class ClusterGrouping extends Grouping {
 
   public class ClusterBuilder extends StageWrapperBuilder {
 
+    /**
+     * Constructor.
+     *
+     * @param grouping
+     *          the upgrade/downgrade grouping (not {@code null}).
+     */
+    private ClusterBuilder(Grouping grouping) {
+      super(grouping);
+    }
+
     @Override
     public void add(UpgradeContext ctx, HostsType hostsType, String service,
         boolean clientOnly, ProcessingComponent pc) {
       // !!! no-op in this case
     }
 
+    /**
+     * {@inheritDoc}
+     */
     @Override
-    public List<StageWrapper> build(UpgradeContext ctx) {
+    public List<StageWrapper> build(UpgradeContext upgradeContext,
+        List<StageWrapper> stageWrappers) {
+
       if (null == executionStages) {
-        return Collections.emptyList();
+        return stageWrappers;
       }
 
-      List<StageWrapper> results = new ArrayList<StageWrapper>();
+      List<StageWrapper> results = new ArrayList<StageWrapper>(stageWrappers);
 
       if (executionStages != null) {
         for (ExecuteStage execution : executionStages) {
-          if (null != execution.intendedDirection &&
-              execution.intendedDirection != ctx.getDirection()) {
+          if (null != execution.intendedDirection
+              && execution.intendedDirection != upgradeContext.getDirection()) {
             continue;
           }
 
@@ -127,7 +138,7 @@ public class ClusterGrouping extends Grouping {
 
           switch (task.getType()) {
             case MANUAL:
-              wrapper = getManualStageWrapper(ctx, execution);
+              wrapper = getManualStageWrapper(upgradeContext, execution);
               break;
 
             case SERVER_ACTION:
@@ -138,7 +149,7 @@ public class ClusterGrouping extends Grouping {
               break;
 
             case EXECUTE:
-              wrapper = getExecuteStageWrapper(ctx, execution);
+              wrapper = getExecuteStageWrapper(upgradeContext, execution);
               break;
 
             default:

ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ColocatedGrouping.java

@@ -54,9 +54,12 @@ public class ColocatedGrouping extends Grouping {
   public Batch batch;
 
 
+  /**
+   * {@inheritDoc}
+   */
   @Override
   public StageWrapperBuilder getBuilder() {
-    return new MultiHomedBuilder(batch, performServiceCheck);
+    return new MultiHomedBuilder(this, batch, performServiceCheck);
   }
 
   private static class MultiHomedBuilder extends StageWrapperBuilder {
@@ -69,7 +72,9 @@ public class ColocatedGrouping extends Grouping {
     private Map<String, List<TaskProxy>> finalBatches = new LinkedHashMap<String, List<TaskProxy>>();
 
 
-    private MultiHomedBuilder(Batch batch, boolean serviceCheck) {
+    private MultiHomedBuilder(Grouping grouping, Batch batch, boolean serviceCheck) {
+      super(grouping);
+
       m_batch = batch;
       m_serviceCheck = serviceCheck;
     }
@@ -143,16 +148,20 @@ public class ColocatedGrouping extends Grouping {
     }
 
 
+    /**
+     * {@inheritDoc}
+     */
     @Override
-    public List<StageWrapper> build(UpgradeContext ctx) {
-      List<StageWrapper> results = new ArrayList<StageWrapper>();
+    public List<StageWrapper> build(UpgradeContext upgradeContext,
+        List<StageWrapper> stageWrappers) {
+      List<StageWrapper> results = new ArrayList<StageWrapper>(stageWrappers);
 
       if (LOG.isDebugEnabled()) {
         LOG.debug("RU initial: {}", initialBatch);
         LOG.debug("RU final: {}", finalBatches);
       }
 
-      List<StageWrapper> befores = fromProxies(ctx.getDirection(), initialBatch);
+      List<StageWrapper> befores = fromProxies(upgradeContext.getDirection(), initialBatch);
       results.addAll(befores);
 
       if (!befores.isEmpty()) {
@@ -160,16 +169,17 @@ public class ColocatedGrouping extends Grouping {
         ManualTask task = new ManualTask();
         task.summary = m_batch.summary;
         task.message = m_batch.message;
-        formatFirstBatch(ctx, task, befores);
+        formatFirstBatch(upgradeContext, task, befores);
 
         StageWrapper wrapper = new StageWrapper(
             StageWrapper.Type.SERVER_SIDE_ACTION,
-            "Validate Partial " + ctx.getDirection().getText(true),
+            "Validate Partial " + upgradeContext.getDirection().getText(true),
             new TaskWrapper(null, null, Collections.<String>emptySet(), task));
+
         results.add(wrapper);
       }
 
-      results.addAll(fromProxies(ctx.getDirection(), finalBatches));
+      results.addAll(fromProxies(upgradeContext.getDirection(), finalBatches));
 
       return results;
     }

ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java

@@ -65,7 +65,7 @@ public class Grouping {
    * Gets the default builder.
    */
   public StageWrapperBuilder getBuilder() {
-    return new DefaultBuilder(performServiceCheck);
+    return new DefaultBuilder(this, performServiceCheck);
   }
 
 
@@ -75,7 +75,8 @@ public class Grouping {
     private Set<String> m_servicesToCheck = new HashSet<String>();
     private boolean m_serviceCheck = true;
 
-    private DefaultBuilder(boolean serviceCheck) {
+    private DefaultBuilder(Grouping grouping, boolean serviceCheck) {
+      super(grouping);
       m_serviceCheck = serviceCheck;
     }
 
@@ -139,23 +140,32 @@ public class Grouping {
       }
     }
 
+    /**
+     * {@inheritDoc}
+     */
     @Override
-    public List<StageWrapper> build(UpgradeContext ctx) {
+    public List<StageWrapper> build(UpgradeContext upgradeContext,
+        List<StageWrapper> stageWrappers) {
+
+      // insert all pre-processed stage wrappers first
+      if (!stageWrappers.isEmpty()) {
+        m_stages.addAll(0, stageWrappers);
+      }
 
       List<TaskWrapper> tasks = new ArrayList<TaskWrapper>();
       List<String> displays = new ArrayList<String>();
       for (String service : m_servicesToCheck) {
         tasks.add(new TaskWrapper(
             service, "", Collections.<String>emptySet(), new ServiceCheckTask()));
-        displays.add(ctx.getServiceDisplay(service));
+
+        displays.add(upgradeContext.getServiceDisplay(service));
       }
 
-      if (ctx.getDirection().isUpgrade() && m_serviceCheck && m_servicesToCheck.size() > 0) {
-        StageWrapper wrapper = new StageWrapper(
-            StageWrapper.Type.SERVICE_CHECK,
-            "Service Check " + StringUtils.join(displays, ", "),
-            tasks.toArray(new TaskWrapper[0])
-            );
+      if (upgradeContext.getDirection().isUpgrade() && m_serviceCheck
+          && m_servicesToCheck.size() > 0) {
+
+        StageWrapper wrapper = new StageWrapper(StageWrapper.Type.SERVICE_CHECK,
+            "Service Check " + StringUtils.join(displays, ", "), tasks.toArray(new TaskWrapper[0]));
 
         m_stages.add(wrapper);
       }
@@ -168,8 +178,9 @@ public class Grouping {
    * Group all like-typed tasks together.  When they change, create a new type.
    */
   private static List<TaskBucket> buckets(List<Task> tasks) {
-    if (null == tasks || tasks.isEmpty())
+    if (null == tasks || tasks.isEmpty()) {
       return Collections.emptyList();
+    }
 
     List<TaskBucket> holders = new ArrayList<TaskBucket>();
 

ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ServerActionTask.java

@@ -17,8 +17,6 @@
  */
 package org.apache.ambari.server.state.stack.upgrade;
 
-import org.apache.ambari.server.serveraction.upgrades.FinalizeUpgradeAction;
-
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlRootElement;

ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ServiceCheckGrouping.java

@@ -58,11 +58,12 @@ public class ServiceCheckGrouping extends Grouping {
   @XmlElement(name="service")
   private Set<String> excludeServices = new HashSet<String>();
 
-  private ServiceCheckBuilder m_builder = new ServiceCheckBuilder();
-
+  /**
+   * {@inheritDoc}
+   */
   @Override
   public ServiceCheckBuilder getBuilder() {
-    return m_builder;
+    return new ServiceCheckBuilder(this);
   }
 
   /**
@@ -76,23 +77,40 @@ public class ServiceCheckGrouping extends Grouping {
    * Used to build stages for service check groupings.
    */
   public class ServiceCheckBuilder extends StageWrapperBuilder {
+
     private Cluster m_cluster;
     private AmbariMetaInfo m_metaInfo;
 
+    /**
+     * Constructor.
+     *
+     * @param grouping
+     *          the upgrade/downgrade grouping (not {@code null}).
+     */
+    protected ServiceCheckBuilder(Grouping grouping) {
+      super(grouping);
+    }
 
+    /**
+     * {@inheritDoc}
+     */
     @Override
     public void add(UpgradeContext ctx, HostsType hostsType, String service,
         boolean clientOnly, ProcessingComponent pc) {
       // !!! nothing to do here
     }
 
+    /**
+     * {@inheritDoc}
+     */
     @Override
-    public List<StageWrapper> build(UpgradeContext ctx) {
-      m_cluster = ctx.getCluster();
-      m_metaInfo = ctx.getAmbariMetaInfo();
+    public List<StageWrapper> build(UpgradeContext upgradeContext,
+        List<StageWrapper> stageWrappers) {
+      m_cluster = upgradeContext.getCluster();
+      m_metaInfo = upgradeContext.getAmbariMetaInfo();
 
-      List<StageWrapper> result = new ArrayList<StageWrapper>();
-      if (ctx.getDirection().isDowngrade()) {
+      List<StageWrapper> result = new ArrayList<StageWrapper>(stageWrappers);
+      if (upgradeContext.getDirection().isDowngrade()) {
         return result;
       }
 
@@ -101,28 +119,29 @@ public class ServiceCheckGrouping extends Grouping {
       Set<String> clusterServices = new LinkedHashSet<String>(serviceMap.keySet());
 
       // create stages for the priorities
-      for (String service : ServiceCheckGrouping.this.priorityServices) {
-        if (checkServiceValidity(ctx, service, serviceMap)) {
+      for (String service : priorityServices) {
+        if (checkServiceValidity(upgradeContext, service, serviceMap)) {
           StageWrapper wrapper = new StageWrapper(
               StageWrapper.Type.SERVICE_CHECK,
-              "Service Check " + ctx.getServiceDisplay(service),
+              "Service Check " + upgradeContext.getServiceDisplay(service),
               new TaskWrapper(service, "", Collections.<String>emptySet(),
                   new ServiceCheckTask()));
-          result.add(wrapper);
 
+          result.add(wrapper);
           clusterServices.remove(service);
         }
       }
 
       // create stages for everything else, as long it is valid
       for (String service : clusterServices) {
-        if (ServiceCheckGrouping.this.excludeServices.contains(service)) {
+        if (excludeServices.contains(service)) {
           continue;
         }
-        if (checkServiceValidity(ctx, service, serviceMap)) {
+
+        if (checkServiceValidity(upgradeContext, service, serviceMap)) {
           StageWrapper wrapper = new StageWrapper(
               StageWrapper.Type.SERVICE_CHECK,
-              "Service Check " + ctx.getServiceDisplay(service),
+              "Service Check " + upgradeContext.getServiceDisplay(service),
               new TaskWrapper(service, "", Collections.<String>emptySet(),
                   new ServiceCheckTask()));
           result.add(wrapper);

ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java

@@ -17,9 +17,12 @@
  */
 package org.apache.ambari.server.state.stack.upgrade;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.ambari.server.serveraction.upgrades.AutoSkipFailedSummaryAction;
 import org.apache.ambari.server.stack.HostsType;
 import org.apache.ambari.server.state.UpgradeContext;
 import org.apache.ambari.server.state.stack.UpgradePack.ProcessingComponent;
@@ -29,24 +32,130 @@ import org.apache.ambari.server.state.stack.UpgradePack.ProcessingComponent;
  */
 public abstract class StageWrapperBuilder {
 
+  /**
+   * The message for the task which checks for skipped failures.
+   */
+  private static final String AUTO_SKIPPED_TASK_SUMMARY = "Pauses the upgrade if there were failed steps that were automatically skipped.";
+
+  /**
+   * The upgrade/downgrade grouping that the builder is for.
+   */
+  protected final Grouping m_grouping;
+
+  /**
+   * Constructor.
+   *
+   * @param grouping
+   *          the upgrade/downgrade grouping (not {@code null}).
+   */
+  protected StageWrapperBuilder(Grouping grouping) {
+    m_grouping = grouping;
+  }
+
   /**
    * Adds a processing component that will be built into stage wrappers.
    *
-   * @param ctx         the upgrade context
-   * @param hostsType   the hosts, along with their type
-   * @param service     the service name
-   * @param clientOnly  whether the service is client only, no service checks
-   * @param pc          the ProcessingComponent derived from the upgrade pack
+   * @param upgradeContext
+   *          the upgrade context
+   * @param hostsType
+   *          the hosts, along with their type
+   * @param service
+   *          the service name
+   * @param clientOnly
+   *          whether the service is client only, no service checks
+   * @param pc
+   *          the ProcessingComponent derived from the upgrade pack
    */
-  public abstract void add(UpgradeContext ctx, HostsType hostsType, String service,
+  public abstract void add(UpgradeContext upgradeContext, HostsType hostsType, String service,
       boolean clientOnly, ProcessingComponent pc);
 
   /**
-   * Builds the stage wrappers.
-   * @param ctx the upgrade context
+   * Builds the stage wrappers, including any pre- and post-procesing that needs
+   * to be performed.
+   *
+   * @param upgradeContext
+   *          the upgrade context (not {@code null}).
    * @return a list of stages, never {@code null}
    */
-  public abstract List<StageWrapper> build(UpgradeContext ctx);
+  public final List<StageWrapper> build(UpgradeContext upgradeContext) {
+    List<StageWrapper> stageWrappers = beforeBuild(upgradeContext);
+    stageWrappers = build(upgradeContext, stageWrappers);
+    stageWrappers = afterBuild(upgradeContext, stageWrappers);
+    return stageWrappers;
+  }
+
+  /**
+   * Performs any pre-processing that needs to be performed on the list of stage
+   * wrappers.
+   *
+   * @param upgradeContext
+   *          the upgrade context (not {@code null}).
+   * @return the initial list of stage wrappers, or an empty list (never
+   *         {@code null}).
+   */
+  protected List<StageWrapper> beforeBuild(UpgradeContext upgradeContext) {
+    List<StageWrapper> stageWrappers = new ArrayList<>(100);
+    return stageWrappers;
+  }
+
+  /**
+   * Builds the stage wrappers.
+   *
+   * @param upgradeContext
+   *          the upgrade context (not {@code null}).
+   * @param stageWrappers
+   *          the list of stage wrappers created by
+   *          {@link #beforeBuild(UpgradeContext)}.
+   * @return the stage wrapper list, (never {@code null})
+   */
+  public abstract List<StageWrapper> build(UpgradeContext upgradeContext,
+      List<StageWrapper> stageWrappers);
+
+  /**
+   * Performs any post-processing that needs to be performed on the list of
+   * stage wrappers.
+   *
+   * @param upgradeContext
+   *          the upgrade context (not {@code null}).
+   * @param stageWrappers
+   *          the list of stage wrappers created by
+   *          {@link #build(UpgradeContext, List)}.
+   * @return the post-processed list of stage wrappers (never {@code null})
+   */
+  protected List<StageWrapper> afterBuild(UpgradeContext upgradeContext,
+      List<StageWrapper> stageWrappers) {
+
+    if (stageWrappers.isEmpty()) {
+      return stageWrappers;
+    }
+
+    // we only want to insert the auto skip summary if the group is skippable
+    // and the upgrade context says to auto skip failures
+    final boolean autoSkipFailures;
+    if (m_grouping instanceof ServiceCheckGrouping) {
+      autoSkipFailures = upgradeContext.isServiceCheckFailureAutoSkipped();
+    } else {
+      autoSkipFailures = upgradeContext.isComponentFailureAutoSkipped();
+    }
+
+    if (m_grouping.skippable && autoSkipFailures) {
+      ServerActionTask skippedFailedCheck = new ServerActionTask();
+      skippedFailedCheck.implClass = AutoSkipFailedSummaryAction.class.getName();
+      skippedFailedCheck.summary = AUTO_SKIPPED_TASK_SUMMARY;
+
+      TaskWrapper skippedFailedTaskWrapper = new TaskWrapper(null, null,
+          Collections.<String> emptySet(), skippedFailedCheck);
+
+      StageWrapper skippedFailedStageWrapper = new StageWrapper(
+          StageWrapper.Type.SERVER_SIDE_ACTION, "Verifying Skipped Failures",
+          skippedFailedTaskWrapper);
+
+      stageWrappers.add(skippedFailedStageWrapper);
+    }
+
+    return stageWrappers;
+  }
+
 
   /**
    * Consistently formats a string.

ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAOTest.java

@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.orm.dao;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.Role;
+import org.apache.ambari.server.RoleCommand;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.orm.GuiceJpaInitializer;
+import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
+import org.apache.ambari.server.orm.OrmTestHelper;
+import org.apache.ambari.server.orm.entities.ClusterEntity;
+import org.apache.ambari.server.orm.entities.HostEntity;
+import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
+import org.apache.ambari.server.orm.entities.RequestEntity;
+import org.apache.ambari.server.orm.entities.StageEntity;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.persist.PersistService;
+
+import junit.framework.Assert;
+
+/**
+ * Tests {@link HostRoleCommandDAO}.
+ */
+public class HostRoleCommandDAOTest {
+
+  private Injector m_injector;
+  private ClusterDAO m_clusterDAO;
+  private StageDAO m_stageDAO;
+  private HostRoleCommandDAO m_hostRoleCommandDAO;
+  private HostDAO m_hostDAO;
+  private RequestDAO m_requestDAO;
+
+  @Before
+  public void setup() throws Exception {
+    m_injector = Guice.createInjector(new InMemoryDefaultTestModule());
+    m_injector.getInstance(GuiceJpaInitializer.class);
+    m_injector.getInstance(AmbariMetaInfo.class);
+
+    m_clusterDAO = m_injector.getInstance(ClusterDAO.class);
+    m_stageDAO = m_injector.getInstance(StageDAO.class);
+    m_hostRoleCommandDAO = m_injector.getInstance(HostRoleCommandDAO.class);
+    m_hostDAO = m_injector.getInstance(HostDAO.class);
+    m_requestDAO = m_injector.getInstance(RequestDAO.class);
+  }
+
+  @After
+  public void teardown() throws AmbariException {
+    m_injector.getInstance(PersistService.class).stop();
+  }
+
+  /**
+   * Tests finding all tasks between a range of stages.
+   */
+  @Test
+  public void testFindTasksBetweenStages() {
+    OrmTestHelper helper = m_injector.getInstance(OrmTestHelper.class);
+    helper.createDefaultData();
+
+    Long requestId = Long.valueOf(100L);
+    ClusterEntity clusterEntity = m_clusterDAO.findByName("test_cluster1");
+
+    RequestEntity requestEntity = new RequestEntity();
+    requestEntity.setRequestId(requestId);
+    requestEntity.setClusterId(clusterEntity.getClusterId());
+    requestEntity.setStages(new ArrayList<StageEntity>());
+    m_requestDAO.create(requestEntity);
+
+    AtomicLong stageId = new AtomicLong(1);
+    HostEntity host = m_hostDAO.findByName("test_host1");
+    host.setHostRoleCommandEntities(new ArrayList<HostRoleCommandEntity>());
+
+    createStage(stageId.getAndIncrement(), 3, host, requestEntity, HostRoleStatus.COMPLETED, false);
+    createStage(stageId.getAndIncrement(), 2, host, requestEntity, HostRoleStatus.SKIPPED_FAILED, false);
+    createStage(stageId.getAndIncrement(), 1, host, requestEntity, HostRoleStatus.ABORTED, false);
+
+    List<HostRoleCommandEntity> tasks = m_hostRoleCommandDAO.findByStatusBetweenStages(requestId,
+        HostRoleStatus.SKIPPED_FAILED, 1, 3);
+
+    Assert.assertEquals(2, tasks.size());
+
+    tasks = m_hostRoleCommandDAO.findByStatusBetweenStages(requestId, HostRoleStatus.SKIPPED_FAILED, 1, 1);
+    Assert.assertEquals(0, tasks.size());
+  }
+
+  /**
+   * Creates a single stage with the specified number of commands.
+   *
+   * @param startStageId
+   * @param count
+   * @param hostEntity
+   * @param requestEntity
+   * @param status
+   * @param skipStage
+   * @return
+   */
+  private void createStage(long startStageId, int count, HostEntity hostEntity,
+      RequestEntity requestEntity, HostRoleStatus status, boolean skipStage) {
+    long stageId = startStageId;
+
+    ClusterEntity clusterEntity = m_clusterDAO.findByName("test_cluster1");
+
+    StageEntity stageEntity = new StageEntity();
+    stageEntity.setClusterId(clusterEntity.getClusterId());
+    stageEntity.setRequest(requestEntity);
+    stageEntity.setStageId(stageId);
+    stageEntity.setHostRoleCommands(new ArrayList<HostRoleCommandEntity>());
+    stageEntity.setSkippable(skipStage);
+    m_stageDAO.create(stageEntity);
+    requestEntity.getStages().add(stageEntity);
+
+    for (int i = 0; i < count; i++) {
+      HostRoleCommandEntity commandEntity = new HostRoleCommandEntity();
+      commandEntity.setRequestId(requestEntity.getRequestId());
+      commandEntity.setStageId(stageId);
+      commandEntity.setRoleCommand(RoleCommand.INSTALL);
+      commandEntity.setStatus(status);
+      commandEntity.setRole(Role.DATANODE);
+      commandEntity.setHostEntity(hostEntity);
+      commandEntity.setStage(stageEntity);
+      m_hostRoleCommandDAO.create(commandEntity);
+
+      hostEntity.getHostRoleCommandEntities().add(commandEntity);
+      hostEntity = m_hostDAO.merge(hostEntity);
+
+      stageEntity.getHostRoleCommands().add(commandEntity);
+      m_stageDAO.merge(stageEntity);
+    }
+  }
+}

ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilderTest.java

@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.state.stack.upgrade;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.ambari.server.serveraction.upgrades.AutoSkipFailedSummaryAction;
+import org.apache.ambari.server.stack.HostsType;
+import org.apache.ambari.server.state.UpgradeContext;
+import org.apache.ambari.server.state.stack.UpgradePack.ProcessingComponent;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests the {@link StageWrapperBuilder}.
+ */
+public class StageWrapperBuilderTest {
+
+
+  /**
+   * Tests that the various build methods of a builder are invoked in the
+   * correct order.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testBuildOrder() throws Exception {
+    UpgradeContext upgradeContext = new UpgradeContext(null, null, null, null, Direction.UPGRADE);
+    MockStageWrapperBuilder builder = new MockStageWrapperBuilder(null);
+    List<StageWrapper> stageWrappers = builder.build(upgradeContext);
+    List<Integer> invocationOrder = builder.getInvocationOrder();
+
+    Assert.assertEquals(Integer.valueOf(0), invocationOrder.get(0));
+    Assert.assertEquals(Integer.valueOf(1), invocationOrder.get(1));
+    Assert.assertEquals(Integer.valueOf(2), invocationOrder.get(2));
+
+    // nothing happened, so this should be empty
+    Assert.assertTrue(stageWrappers.isEmpty());
+  }
+
+  /**
+   * Tests that a new task was inserted into the upgrade which will check for
+   * skipped failures and display a summary.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testAutoSkipCheckInserted() throws Exception {
+    UpgradeContext upgradeContext = new UpgradeContext(null, null, null, null, Direction.UPGRADE);
+    upgradeContext.setAutoSkipComponentFailures(true);
+    upgradeContext.setAutoSkipServiceCheckFailures(true);
+
+    Grouping grouping = new Grouping();
+    grouping.skippable = true;
+
+    MockStageWrapperBuilder builder = new MockStageWrapperBuilder(grouping);
+
+    List<StageWrapper> mockStageWrappers = new ArrayList<>();
+    StageWrapper mockStageWrapper = EasyMock.createNiceMock(StageWrapper.class);
+    mockStageWrappers.add(mockStageWrapper);
+
+    builder.setMockStageWrappers(mockStageWrappers);
+
+    List<StageWrapper> stageWrappers = builder.build(upgradeContext);
+    Assert.assertEquals(2, stageWrappers.size());
+
+    StageWrapper skipSummaryWrapper = stageWrappers.get(1);
+    Assert.assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, skipSummaryWrapper.getType());
+
+    ServerActionTask task = (ServerActionTask)(skipSummaryWrapper.getTasks().get(0).getTasks().get(0));
+    Assert.assertEquals(AutoSkipFailedSummaryAction.class.getName(), task.implClass);
+
+  }
+
+  /**
+   * A mock {@link StageWrapperBuilder}.
+   */
+  private final class MockStageWrapperBuilder extends StageWrapperBuilder {
+
+    private List<Integer> m_invocationOrder = new ArrayList<>();
+    private List<StageWrapper> m_stageWrappers = Collections.emptyList();
+
+    /**
+     * Constructor.
+     *
+     * @param grouping
+     */
+    protected MockStageWrapperBuilder(Grouping grouping) {
+      super(grouping);
+    }
+
+    private void setMockStageWrappers(List<StageWrapper> stageWrappers) {
+      m_stageWrappers = stageWrappers;
+    }
+
+    /**
+     * Gets the invocation order.
+     *
+     * @return
+     */
+    private List<Integer> getInvocationOrder() {
+      return m_invocationOrder;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public void add(UpgradeContext upgradeContext, HostsType hostsType, String service,
+        boolean clientOnly, ProcessingComponent pc) {
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public List<StageWrapper> build(UpgradeContext upgradeContext,
+        List<StageWrapper> stageWrappers) {
+      m_invocationOrder.add(1);
+      return m_stageWrappers;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    protected List<StageWrapper> beforeBuild(UpgradeContext upgradeContext) {
+      List<StageWrapper> stageWrappers = super.beforeBuild(upgradeContext);
+      m_invocationOrder.add(0);
+      return stageWrappers;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    protected List<StageWrapper> afterBuild(UpgradeContext upgradeContext,
+        List<StageWrapper> stageWrappers) {
+      stageWrappers = super.afterBuild(upgradeContext, stageWrappers);
+      m_invocationOrder.add(2);
+      return stageWrappers;
+    }
+  }
+}