AMBARI-13524 - Change a Request In Progress To Skip Errors Automatically (jonathanhurley)

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java

@@ -21,6 +21,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOL
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.VERSION;
 
+import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -102,10 +103,12 @@ import org.apache.ambari.server.state.stack.upgrade.TaskWrapper;
 import org.apache.ambari.server.state.stack.upgrade.UpdateStackGrouping;
 import org.apache.ambari.server.state.stack.upgrade.UpgradeType;
 import org.apache.ambari.server.state.svccomphost.ServiceComponentHostServerActionEvent;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.collect.Lists;
 import com.google.gson.Gson;
 import com.google.inject.Inject;
 import com.google.inject.Provider;
@@ -387,57 +390,68 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
     // !!! above check ensures only one
     final Map<String, Object> propertyMap = requestMaps.iterator().next();
 
-    String requestId = (String) propertyMap.get(UPGRADE_REQUEST_ID);
-    if (null == requestId) {
+    String requestIdProperty = (String) propertyMap.get(UPGRADE_REQUEST_ID);
+    if (null == requestIdProperty) {
       throw new IllegalArgumentException(String.format("%s is required", UPGRADE_REQUEST_ID));
     }
 
-    String requestStatus = (String) propertyMap.get(UPGRADE_REQUEST_STATUS);
-    if (null == requestStatus) {
-      throw new IllegalArgumentException(String.format("%s is required", UPGRADE_REQUEST_STATUS));
+    long requestId = Long.parseLong(requestIdProperty);
+    UpgradeEntity upgradeEntity = s_upgradeDAO.findUpgradeByRequestId(requestId);
+    if( null == upgradeEntity){
+      String exceptionMessage = MessageFormat.format("The upgrade with request ID {0} was not found", requestIdProperty);
+      throw new NoSuchParentResourceException(exceptionMessage);
     }
 
-    HostRoleStatus status = HostRoleStatus.valueOf(requestStatus);
-    if (status != HostRoleStatus.ABORTED && status != HostRoleStatus.PENDING) {
-      throw new IllegalArgumentException(String.format("Cannot set status %s, only %s is allowed",
-          status, EnumSet.of(HostRoleStatus.ABORTED, HostRoleStatus.PENDING)));
-    }
+    // the properties which are allowed to be updated; the request must include
+    // at least 1
+    List<String> updatableProperties = Lists.newArrayList(UPGRADE_REQUEST_STATUS,
+        UPGRADE_SKIP_FAILURES, UPGRADE_SKIP_SC_FAILURES);
 
-    String reason = (String) propertyMap.get(UPGRADE_ABORT_REASON);
-    if (null == reason) {
-      reason = String.format(DEFAULT_REASON_TEMPLATE, requestId);
-    }
+    boolean isRequiredPropertyInRequest = CollectionUtils.containsAny(updatableProperties,
+        propertyMap.keySet());
 
-    ActionManager actionManager = getManagementController().getActionManager();
-    List<org.apache.ambari.server.actionmanager.Request> requests = actionManager.getRequests(
-        Collections.singletonList(Long.valueOf(requestId)));
-
-    org.apache.ambari.server.actionmanager.Request internalRequest = requests.get(0);
+    if (!isRequiredPropertyInRequest) {
+      String exceptionMessage = MessageFormat.format(
+          "At least one of the following properties is required in the request: {0}",
+          StringUtils.join(updatableProperties, ", "));
+      throw new IllegalArgumentException(exceptionMessage);
+    }
 
-    HostRoleStatus internalStatus = CalculatedStatus.statusFromStages(
-        internalRequest.getStages()).getStatus();
+    String requestStatus = (String) propertyMap.get(UPGRADE_REQUEST_STATUS);
+    String skipFailuresRequestProperty = (String) propertyMap.get(UPGRADE_SKIP_FAILURES);
+    String skipServiceCheckFailuresRequestProperty = (String) propertyMap.get(UPGRADE_SKIP_SC_FAILURES);
 
-    if (HostRoleStatus.PENDING == status && internalStatus != HostRoleStatus.ABORTED) {
-      throw new IllegalArgumentException(
-          String.format("Can only set status to %s when the upgrade is %s (currently %s)", status,
-              HostRoleStatus.ABORTED, internalStatus));
+    if (null != requestStatus) {
+      HostRoleStatus status = HostRoleStatus.valueOf(requestStatus);
+      setUpgradeRequestStatus(requestIdProperty, status, propertyMap);
     }
 
-    if (HostRoleStatus.ABORTED == status) {
-      if (!internalStatus.isCompletedState()) {
-        actionManager.cancelRequest(internalRequest.getRequestId(), reason);
+    // if either of the skip failure settings are in the request, then we need
+    // to iterate over the entire series of tasks anyway, so do them both at the
+    // same time
+    if (StringUtils.isNotEmpty(skipFailuresRequestProperty)
+        || StringUtils.isNotEmpty(skipServiceCheckFailuresRequestProperty)) {
+      // grab the current settings for both
+      boolean skipFailures = upgradeEntity.isComponentFailureAutoSkipped();
+      boolean skipServiceCheckFailures = upgradeEntity.isServiceCheckFailureAutoSkipped();
+
+      // update skipping failures on commands which are not SERVICE_CHECKs
+      if (null != skipFailuresRequestProperty) {
+        skipFailures = Boolean.parseBoolean(skipFailuresRequestProperty);
+        s_hostRoleCommandDAO.updateAutomaticSkipOnFailure(requestId, skipFailures);
       }
-    } else {
-      List<Long> taskIds = new ArrayList<Long>();
 
-      for (HostRoleCommand hrc : internalRequest.getCommands()) {
-        if (HostRoleStatus.ABORTED == hrc.getStatus()
-            || HostRoleStatus.TIMEDOUT == hrc.getStatus()) {
-          taskIds.add(hrc.getTaskId());
-        }
+      // if the service check failure skip is present, then update all role
+      // commands that are SERVICE_CHECKs
+      if (null != skipServiceCheckFailuresRequestProperty) {
+        skipServiceCheckFailures = Boolean.parseBoolean(skipServiceCheckFailuresRequestProperty);
+        s_hostRoleCommandDAO.updateAutomaticSkipServiceCheckFailure(requestId,
+            skipServiceCheckFailures);
       }
 
-      actionManager.resubmitTasks(taskIds);
+      upgradeEntity.setAutoSkipComponentFailures(skipFailures);
+      upgradeEntity.setAutoSkipServiceCheckFailures(skipServiceCheckFailures);
+      upgradeEntity = s_upgradeDAO.merge(upgradeEntity);
     }
 
     return getRequestStatus(null);
@@ -1449,4 +1463,65 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
     parameters.put(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION, "*");
     return parameters;
   }
+
+  /**
+   * Changes the status of the specified request for an upgrade. The valid
+   * values are:
+   * <ul>
+   * <li>{@link HostRoleStatus#ABORTED}</li>
+   * <li>{@link HostRoleStatus#PENDING}</li>
+   * </ul>
+   *
+   * @param requestId
+   *          the request to change the status for.
+   * @param status
+   *          the status to set
+   * @param propertyMap
+   *          the map of request properties (needed for things like abort reason
+   *          if present)
+   */
+  private void setUpgradeRequestStatus(String requestId, HostRoleStatus status,
+      Map<String, Object> propertyMap) {
+    if (status != HostRoleStatus.ABORTED && status != HostRoleStatus.PENDING) {
+      throw new IllegalArgumentException(String.format("Cannot set status %s, only %s is allowed",
+          status, EnumSet.of(HostRoleStatus.ABORTED, HostRoleStatus.PENDING)));
+    }
+
+    String reason = (String) propertyMap.get(UPGRADE_ABORT_REASON);
+    if (null == reason) {
+      reason = String.format(DEFAULT_REASON_TEMPLATE, requestId);
+    }
+
+    ActionManager actionManager = getManagementController().getActionManager();
+    List<org.apache.ambari.server.actionmanager.Request> requests = actionManager.getRequests(
+        Collections.singletonList(Long.valueOf(requestId)));
+
+    org.apache.ambari.server.actionmanager.Request internalRequest = requests.get(0);
+
+    HostRoleStatus internalStatus = CalculatedStatus.statusFromStages(
+        internalRequest.getStages()).getStatus();
+
+    if (HostRoleStatus.PENDING == status && internalStatus != HostRoleStatus.ABORTED) {
+      throw new IllegalArgumentException(
+          String.format("Can only set status to %s when the upgrade is %s (currently %s)", status,
+              HostRoleStatus.ABORTED, internalStatus));
+    }
+
+    if (HostRoleStatus.ABORTED == status) {
+      if (!internalStatus.isCompletedState()) {
+        actionManager.cancelRequest(internalRequest.getRequestId(), reason);
+      }
+    } else {
+      List<Long> taskIds = new ArrayList<Long>();
+
+      for (HostRoleCommand hrc : internalRequest.getCommands()) {
+        if (HostRoleStatus.ABORTED == hrc.getStatus()
+            || HostRoleStatus.TIMEDOUT == hrc.getStatus()) {
+          taskIds.add(hrc.getTaskId());
+        }
+      }
+
+      actionManager.resubmitTasks(taskIds);
+    }
+  }
 }

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java

@@ -32,6 +32,7 @@ import java.util.Map;
 import javax.persistence.EntityManager;
 import javax.persistence.TypedQuery;
 
+import org.apache.ambari.server.RoleCommand;
 import org.apache.ambari.server.actionmanager.HostRoleStatus;
 import org.apache.ambari.server.orm.RequiresSession;
 import org.apache.ambari.server.orm.entities.HostEntity;
@@ -474,4 +475,70 @@ public class HostRoleCommandDAO {
 
     return map;
   }
+
+  /**
+   * Updates the {@link HostRoleCommandEntity#isFailureAutoSkipped()} flag for
+   * all commands which are not {@link RoleCommand#SERVICE_CHECK}.
+   * <p/>
+   * This will execute a JPQL {@code UPDATE} statement, bypassing the
+   * {@link EntityManager}. It does this because the amount of
+   * {@link HostRoleCommandEntity}s could number in the 10's of 1000's. As a
+   * result, this will call {@link EntityManager#clear()} after the update to
+   * ensure that the updated entity state is reflected in future queries.
+   *
+   * @param requestId
+   *          the request ID of the commands to update
+   * @param automaticallySkipOnFailure
+   *          {@code true} to automatically skip failures, {@code false}
+   *          otherwise.
+   * @see HostRoleCommandDAO#updateAutomaticSkipServiceCheckFailure(long,
+   *      boolean)
+   */
+  @Transactional
+  public void updateAutomaticSkipOnFailure(long requestId, boolean automaticallySkipOnFailure) {
+    EntityManager entityManager = entityManagerProvider.get();
+
+    TypedQuery<HostRoleCommandEntity> query = entityManager.createNamedQuery(
+        "HostRoleCommandEntity.updateAutoSkipExcludeRoleCommand", HostRoleCommandEntity.class);
+
+    query.setParameter("requestId", requestId);
+    query.setParameter("roleCommand", RoleCommand.SERVICE_CHECK);
+    query.setParameter("autoSkipOnFailure", automaticallySkipOnFailure ? 1 : 0);
+    query.executeUpdate();
+
+    entityManager.clear();
+  }
+
+  /**
+   * Updates the {@link HostRoleCommandEntity#isFailureAutoSkipped()} flag for
+   * all commands which are of type {@link RoleCommand#SERVICE_CHECK}.
+   * <p/>
+   * This will execute a JPQL {@code UPDATE} statement, bypassing the
+   * {@link EntityManager}. It does this because the amount of
+   * {@link HostRoleCommandEntity}s could number in the 10's of 1000's. As a
+   * result, this will call {@link EntityManager#clear()} after the update to
+   * ensure that the updated entity state is reflected in future queries.
+   *
+   * @param requestId
+   *          the request ID of the service check commands to update
+   * @param automaticallySkipOnFailure
+   *          {@code true} to automatically skip service check failures,
+   *          {@code false} otherwise.
+   * @see HostRoleCommandDAO#updateAutomaticSkipOnFailure(long, boolean)
+   */
+  @Transactional
+  public void updateAutomaticSkipServiceCheckFailure(long requestId,
+      boolean automaticallySkipOnFailure) {
+    EntityManager entityManager = entityManagerProvider.get();
+
+    TypedQuery<HostRoleCommandEntity> query = entityManager.createNamedQuery(
+        "HostRoleCommandEntity.updateAutoSkipForRoleCommand", HostRoleCommandEntity.class);
+
+    query.setParameter("requestId", requestId);
+    query.setParameter("roleCommand", RoleCommand.SERVICE_CHECK);
+    query.setParameter("autoSkipOnFailure", automaticallySkipOnFailure ? 1 : 0);
+    query.executeUpdate();
+
+    entityManager.clear();
+  }
 }

ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity.java

@@ -60,7 +60,9 @@ import org.apache.commons.lang.ArrayUtils;
     @NamedQuery(name = "HostRoleCommandEntity.findByHostId", query = "SELECT command FROM HostRoleCommandEntity command WHERE command.hostId=:hostId"),
     @NamedQuery(name = "HostRoleCommandEntity.findByHostRole", query = "SELECT command FROM HostRoleCommandEntity command WHERE command.hostEntity.hostName=:hostName AND command.requestId=:requestId AND command.stageId=:stageId AND command.role=:role ORDER BY command.taskId"),
     @NamedQuery(name = "HostRoleCommandEntity.findByHostRoleNullHost", query = "SELECT command FROM HostRoleCommandEntity command WHERE command.hostEntity IS NULL AND command.requestId=:requestId AND command.stageId=:stageId AND command.role=:role"),
-    @NamedQuery(name = "HostRoleCommandEntity.findByStatusBetweenStages", query = "SELECT command FROM HostRoleCommandEntity command WHERE command.requestId = :requestId AND command.stageId >= :minStageId AND command.stageId <= :maxStageId AND command.status = :status")
+    @NamedQuery(name = "HostRoleCommandEntity.findByStatusBetweenStages", query = "SELECT command FROM HostRoleCommandEntity command WHERE command.requestId = :requestId AND command.stageId >= :minStageId AND command.stageId <= :maxStageId AND command.status = :status"),
+    @NamedQuery(name = "HostRoleCommandEntity.updateAutoSkipExcludeRoleCommand", query = "UPDATE HostRoleCommandEntity command SET command.autoSkipOnFailure = :autoSkipOnFailure WHERE command.requestId = :requestId AND command.roleCommand <> :roleCommand"),
+    @NamedQuery(name = "HostRoleCommandEntity.updateAutoSkipForRoleCommand", query = "UPDATE HostRoleCommandEntity command SET command.autoSkipOnFailure = :autoSkipOnFailure WHERE command.requestId = :requestId AND command.roleCommand = :roleCommand")
 })
 public class HostRoleCommandEntity {
 

ambari-server/src/test/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAOTest.java

@@ -108,6 +108,46 @@ public class HostRoleCommandDAOTest {
     Assert.assertEquals(0, tasks.size());
   }
 
+  /**
+   * Tests updating various commands to be skipped on failures automatically.
+   */
+  @Test
+  public void testUpdateAutoSkipOnFailures() {
+    OrmTestHelper helper = m_injector.getInstance(OrmTestHelper.class);
+    helper.createDefaultData();
+
+    Long requestId = Long.valueOf(100L);
+    ClusterEntity clusterEntity = m_clusterDAO.findByName("test_cluster1");
+
+    RequestEntity requestEntity = new RequestEntity();
+    requestEntity.setRequestId(requestId);
+    requestEntity.setClusterId(clusterEntity.getClusterId());
+    requestEntity.setStages(new ArrayList<StageEntity>());
+    m_requestDAO.create(requestEntity);
+
+    AtomicLong stageId = new AtomicLong(1);
+    HostEntity host = m_hostDAO.findByName("test_host1");
+    host.setHostRoleCommandEntities(new ArrayList<HostRoleCommandEntity>());
+
+    createStage(stageId.getAndIncrement(), 3, host, requestEntity, HostRoleStatus.PENDING, false);
+    createStage(stageId.getAndIncrement(), 2, host, requestEntity, HostRoleStatus.PENDING, false);
+    createStage(stageId.getAndIncrement(), 1, host, requestEntity, HostRoleStatus.PENDING, false);
+
+    List<HostRoleCommandEntity> tasks = m_hostRoleCommandDAO.findByRequest(requestId);
+    Assert.assertEquals(6, tasks.size());
+
+    for (HostRoleCommandEntity task : tasks) {
+      Assert.assertFalse(task.isFailureAutoSkipped());
+    }
+
+    m_hostRoleCommandDAO.updateAutomaticSkipOnFailure(requestId, true);
+    tasks = m_hostRoleCommandDAO.findByRequest(requestId);
+
+    for (HostRoleCommandEntity task : tasks) {
+      Assert.assertTrue(task.isFailureAutoSkipped());
+    }
+  }
+
   /**
    * Creates a single stage with the specified number of commands.
    *