AMBARI-15482 - Restarting HDFS Before Upgrade Finalizing Does Not Supply the rollingUpgrade Flag (jonathanhurley)

ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java

@@ -387,6 +387,11 @@ public class ExecutionCommand extends AgentCommand {
      * The key indicating that the package_version string is available
      */
     String PACKAGE_VERSION = "package_version";
+
+    /**
+     * The key indicating that there is an un-finalized upgrade which is suspended.
+     */
+    String UPGRADE_SUSPENDED = "upgrade_suspended";
   }
 
 }

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java

@@ -18,10 +18,23 @@
 
 package org.apache.ambari.server.controller;
 
-import com.google.gson.JsonArray;
-import com.google.gson.JsonObject;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_COUNT;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_ON_UNAVAILABILITY;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMPONENT_CATEGORY;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.ObjectNotFoundException;
 import org.apache.ambari.server.Role;
@@ -51,22 +64,10 @@ import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_COUNT;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_ON_UNAVAILABILITY;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMPONENT_CATEGORY;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
 
 /**
  * Helper class containing logic to process custom action execution requests
@@ -435,6 +436,12 @@ public class AmbariActionExecutionHelper {
         roleParams.put(COMPONENT_CATEGORY, componentInfo.getCategory());
       }
 
+      // if there is a stack upgrade which is currently suspended then pass that
+      // information down with the command as some components may need to know
+      if (null != cluster && cluster.isUpgradeSuspended()) {
+        roleParams.put(KeyNames.UPGRADE_SUSPENDED, Boolean.TRUE.toString().toLowerCase());
+      }
+
       execCmd.setRoleParams(roleParams);
 
       // ensure that any tags that need to be refreshed are extracted from the

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java

@@ -18,12 +18,44 @@
 
 package org.apache.ambari.server.controller;
 
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_COUNT;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_ON_UNAVAILABILITY;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMPONENT_CATEGORY;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CUSTOM_COMMAND;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MYSQL_JDBC_URL;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.NOT_MANAGED_HDFS_PATH_LIST;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_LIST;
+
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.Role;
 import org.apache.ambari.server.RoleCommand;
@@ -70,43 +102,12 @@ import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.text.MessageFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_COUNT;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_ON_UNAVAILABILITY;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMPONENT_CATEGORY;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CUSTOM_COMMAND;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_NAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MYSQL_JDBC_URL;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.NOT_MANAGED_HDFS_PATH_LIST;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_LIST;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
 
 /**
  * Helper class containing logic to process custom command execution requests .
@@ -431,6 +432,12 @@ public class AmbariCustomCommandExecutionHelper {
         roleParams = new TreeMap<String, String>();
       }
 
+      // if there is a stack upgrade which is currently suspended then pass that
+      // information down with the command as some components may need to know
+      if (cluster.isUpgradeSuspended()) {
+        roleParams.put(KeyNames.UPGRADE_SUSPENDED, Boolean.TRUE.toString().toLowerCase());
+      }
+
       roleParams.put(COMPONENT_CATEGORY, componentInfo.getCategory());
       execCmd.setRoleParams(roleParams);
     }

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java

@@ -84,7 +84,6 @@ import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Config;
 import org.apache.ambari.server.state.ConfigHelper;
 import org.apache.ambari.server.state.DesiredConfig;
-import org.apache.ambari.server.state.RepositoryType;
 import org.apache.ambari.server.state.Service;
 import org.apache.ambari.server.state.ServiceComponent;
 import org.apache.ambari.server.state.ServiceInfo;
@@ -138,6 +137,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
   protected static final String UPGRADE_DIRECTION = "Upgrade/direction";
   protected static final String UPGRADE_DOWNGRADE_ALLOWED = "Upgrade/downgrade_allowed";
   protected static final String UPGRADE_REQUEST_STATUS = "Upgrade/request_status";
+  protected static final String UPGRADE_SUSPENDED = "Upgrade/suspended";
   protected static final String UPGRADE_ABORT_REASON = "Upgrade/abort_reason";
   protected static final String UPGRADE_SKIP_PREREQUISITE_CHECKS = "Upgrade/skip_prerequisite_checks";
   protected static final String UPGRADE_FAIL_ON_CHECK_WARNINGS = "Upgrade/fail_on_check_warnings";
@@ -255,6 +255,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
     PROPERTY_IDS.add(UPGRADE_TO_VERSION);
     PROPERTY_IDS.add(UPGRADE_DIRECTION);
     PROPERTY_IDS.add(UPGRADE_DOWNGRADE_ALLOWED);
+    PROPERTY_IDS.add(UPGRADE_SUSPENDED);
     PROPERTY_IDS.add(UPGRADE_SKIP_FAILURES);
     PROPERTY_IDS.add(UPGRADE_SKIP_SC_FAILURES);
     PROPERTY_IDS.add(UPGRADE_SKIP_MANUAL_VERIFICATION);
@@ -453,7 +454,24 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
 
     if (null != requestStatus) {
       HostRoleStatus status = HostRoleStatus.valueOf(requestStatus);
+
+      // when aborting an upgrade, the suspend flag must be present to indicate
+      // if the upgrade is merely being suspended
+      boolean suspended = false;
+      if( status == HostRoleStatus.ABORTED && !propertyMap.containsKey(UPGRADE_SUSPENDED)){
+        throw new IllegalArgumentException(String.format(
+            "When changing the state of an upgrade to %s, the %s property is required to be either true or false.",
+            status, UPGRADE_SUSPENDED ));
+      } else {
+        suspended = Boolean.valueOf((String) propertyMap.get(UPGRADE_SUSPENDED));
+      }
+
       setUpgradeRequestStatus(requestIdProperty, status, propertyMap);
+
+      // when the status of the upgrade's request is changing, we also update
+      // the suspended flag
+      upgradeEntity.setSuspended(suspended);
+      s_upgradeDAO.merge(upgradeEntity);
     }
 
     // if either of the skip failure settings are in the request, then we need
@@ -1598,7 +1616,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
    * @param requestId
    *          the request to change the status for.
    * @param status
-   *          the status to set
+   *          the status to set on the associated request.
    * @param propertyMap
    *          the map of request properties (needed for things like abort reason
    *          if present)
@@ -1642,9 +1660,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
           LOG.warn("Could not clear upgrade entity for cluster with id {}", clusterId, e);
         }
       }
-    } else { // Processing PENDING
+    } else {
       List<Long> taskIds = new ArrayList<Long>();
-
       for (HostRoleCommand hrc : internalRequest.getCommands()) {
         if (HostRoleStatus.ABORTED == hrc.getStatus()
             || HostRoleStatus.TIMEDOUT == hrc.getStatus()) {
@@ -1661,7 +1678,6 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
       } catch (AmbariException e) {
         LOG.warn("Could not clear upgrade entity for cluster with id {}", clusterId, e);
       }
-
     }
   }
 }

ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeEntity.java

@@ -94,10 +94,15 @@ public class UpgradeEntity {
   @Column(name="downgrade_allowed", nullable = false)
   private Short downgrade_allowed = 1;
 
+  /**
+   * {@code true} if the upgrade has been marked as suspended.
+   */
+  @Column(name = "suspended", nullable = false, length = 1)
+  private Short suspended = 0;
+
   @OneToMany(mappedBy = "upgradeEntity", cascade = { CascadeType.ALL })
   private List<UpgradeGroupEntity> upgradeGroupEntities;
 
-
   /**
    * @return the id
    */
@@ -217,7 +222,7 @@ public class UpgradeEntity {
    * @param canDowngrade {@code true} to allow downgrade, {@code false} to disallow downgrade
    */
   public void setDowngradeAllowed(boolean canDowngrade) {
-    this.downgrade_allowed = (!canDowngrade ? (short)0 : (short)1);
+    downgrade_allowed = (!canDowngrade ? (short)0 : (short)1);
   }
 
   /**
@@ -287,6 +292,26 @@ public class UpgradeEntity {
     skipServiceCheckFailures = autoSkipServiceCheckFailures ? 1 : 0;
   }
 
+  /**
+   * Gets whether the upgrade is suspended. A suspended upgrade will appear to
+   * have its request aborted, but the intent is to resume it at a later point.
+   *
+   * @return {@code true} if the upgrade is suspended.
+   */
+  public boolean isSuspended() {
+    return suspended != 0;
+  }
+
+  /**
+   * Sets whether the upgrade is suspended.
+   *
+   * @param suspended
+   *          {@code true} to mark the upgrade as suspended.
+   */
+  public void setSuspended(boolean suspended) {
+    this.suspended = suspended ? (short) 1 : (short) 0;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
@@ -316,6 +341,9 @@ public class UpgradeEntity {
     if (direction != null ? !direction.equals(that.direction) : that.direction != null) {
       return false;
     }
+    if (suspended != null ? !suspended.equals(that.suspended) : that.suspended != null) {
+      return false;
+    }
     if (upgradeType != null ? !upgradeType.equals(that.upgradeType) : that.upgradeType != null) {
       return false;
     }
@@ -334,6 +362,7 @@ public class UpgradeEntity {
     result = 31 * result + (fromVersion != null ? fromVersion.hashCode() : 0);
     result = 31 * result + (toVersion != null ? toVersion.hashCode() : 0);
     result = 31 * result + (direction != null ? direction.hashCode() : 0);
+    result = 31 * result + (suspended != null ? suspended.hashCode() : 0);
     result = 31 * result + (upgradeType != null ? upgradeType.hashCode() : 0);
     result = 31 * result + (upgradePackage != null ? upgradePackage.hashCode() : 0);
     return result;

ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java

@@ -652,4 +652,13 @@ public interface Cluster {
    * @throws AmbariException
    */
   void setUpgradeEntity(UpgradeEntity upgradeEntity) throws AmbariException;
+
+  /**
+   * Gets whether there is an upgrade which has been suspended and not yet
+   * finalized.
+   *
+   * @return {@code true} if the last upgrade is in the
+   *         {@link UpgradeState#SUSPENDED}.
+   */
+  boolean isUpgradeSuspended();
 }

ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java

@@ -3489,4 +3489,17 @@ public class ClusterImpl implements Cluster {
       clusterGlobalLock.writeLock().unlock();
     }
   }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public boolean isUpgradeSuspended() {
+    UpgradeEntity lastUpgradeItemForCluster = upgradeDAO.findLastUpgradeForCluster(clusterId);
+    if (null != lastUpgradeItemForCluster) {
+      return lastUpgradeItemForCluster.isSuspended();
+    }
+
+    return false;
+  }
 }

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java

@@ -18,12 +18,15 @@
 
 package org.apache.ambari.server.upgrade;
 
-import com.google.gson.Gson;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import com.google.gson.reflect.TypeToken;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
+import java.io.File;
+import java.io.FileReader;
+import java.lang.reflect.Type;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.controller.AmbariManagementController;
@@ -46,14 +49,12 @@ import org.apache.ambari.server.utils.VersionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
-import java.io.FileReader;
-import java.lang.reflect.Type;
-import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.google.common.reflect.TypeToken;
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
 
 /**
  * Upgrade catalog for version 2.2.2.
@@ -79,6 +80,9 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
   private static final String ATLAS_SERVER_HTTPS_PORT_PROPERTY = "atlas.server.https.port";
   private static final String ATLAS_REST_ADDRESS_PROPERTY = "atlas.rest.address";
 
+  private static final String UPGRADE_TABLE = "upgrade";
+  private static final String UPGRADE_SUSPENDED_COLUMN = "suspended";
+
   private static final String HOST_AGGREGATOR_DAILY_CHECKPOINTCUTOFFMULTIPIER_PROPERTY =
     "timeline.metrics.host.aggregator.daily.checkpointCutOffMultiplier";
   private static final String CLUSTER_AGGREGATOR_DAILY_CHECKPOINTCUTOFFMULTIPIER_PROPERTY =
@@ -154,6 +158,8 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
     dbAccessor.addColumn("topology_host_info", columnInfo);
     dbAccessor.addFKConstraint("topology_host_info", "FK_hostinfo_host_id", "host_id", "hosts", "host_id", true);
     dbAccessor.executeUpdate("update topology_host_info set host_id = (select hosts.host_id from hosts where hosts.host_name = topology_host_info.fqdn)");
+
+    updateUpgradeTable();
   }
 
   @Override
@@ -498,4 +504,18 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
     ));
   }
 
+  /**
+   * Updates the {@value #UPGRADE_TABLE} in the following ways:
+   * <ul>
+   * <li>{value {@link #UPGRADE_SUSPENDED_COLUMN} is added</li>
+   * </ul>
+   *
+   * @throws AmbariException
+   * @throws SQLException
+   */
+  protected void updateUpgradeTable() throws AmbariException, SQLException {
+    dbAccessor.addColumn(UPGRADE_TABLE,
+        new DBAccessor.DBColumnInfo(UPGRADE_SUSPENDED_COLUMN, Short.class, 1, 0, false));
+  }
+
 }

ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql

@@ -689,6 +689,7 @@ CREATE TABLE upgrade (
   skip_failures SMALLINT DEFAULT 0 NOT NULL,
   skip_sc_failures SMALLINT DEFAULT 0 NOT NULL,
   downgrade_allowed SMALLINT DEFAULT 1 NOT NULL,
+  suspended SMALLINT DEFAULT 0 NOT NULL,
   PRIMARY KEY (upgrade_id),
   FOREIGN KEY (cluster_id) REFERENCES clusters(cluster_id),
   FOREIGN KEY (request_id) REFERENCES request(request_id)

ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql

@@ -698,6 +698,7 @@ CREATE TABLE upgrade (
   skip_failures TINYINT(1) NOT NULL DEFAULT 0,
   skip_sc_failures TINYINT(1) NOT NULL DEFAULT 0,
   downgrade_allowed TINYINT(1) NOT NULL DEFAULT 1,
+  suspended TINYINT(1) DEFAULT 0 NOT NULL,
   PRIMARY KEY (upgrade_id),
   FOREIGN KEY (cluster_id) REFERENCES clusters(cluster_id),
   FOREIGN KEY (request_id) REFERENCES request(request_id)

ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql

@@ -687,6 +687,7 @@ CREATE TABLE upgrade (
   skip_failures NUMBER(1) DEFAULT 0 NOT NULL,
   skip_sc_failures NUMBER(1) DEFAULT 0 NOT NULL,
   downgrade_allowed NUMBER(1) DEFAULT 1 NOT NULL,
+  suspended NUMBER(1) DEFAULT 0 NOT NULL,
   PRIMARY KEY (upgrade_id),
   FOREIGN KEY (cluster_id) REFERENCES clusters(cluster_id),
   FOREIGN KEY (request_id) REFERENCES request(request_id)

ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql

@@ -691,6 +691,7 @@ CREATE TABLE upgrade (
   skip_failures SMALLINT DEFAULT 0 NOT NULL,
   skip_sc_failures SMALLINT DEFAULT 0 NOT NULL,
   downgrade_allowed SMALLINT DEFAULT 1 NOT NULL,
+  suspended SMALLINT DEFAULT 0 NOT NULL,
   PRIMARY KEY (upgrade_id),
   FOREIGN KEY (cluster_id) REFERENCES clusters(cluster_id),
   FOREIGN KEY (request_id) REFERENCES request(request_id)

ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql

@@ -772,6 +772,7 @@ CREATE TABLE ambari.upgrade (
   skip_failures SMALLINT DEFAULT 0 NOT NULL,
   skip_sc_failures SMALLINT DEFAULT 0 NOT NULL,
   downgrade_allowed SMALLINT DEFAULT 1 NOT NULL,
+  suspended SMALLINT DEFAULT 0 NOT NULL,
   PRIMARY KEY (upgrade_id),
   FOREIGN KEY (cluster_id) REFERENCES ambari.clusters(cluster_id),
   FOREIGN KEY (request_id) REFERENCES ambari.request(request_id)

ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql

@@ -686,6 +686,7 @@ CREATE TABLE upgrade (
   skip_failures BIT NOT NULL DEFAULT 0,
   skip_sc_failures BIT NOT NULL DEFAULT 0,
   downgrade_allowed BIT NOT NULL DEFAULT 1,
+  suspended BIT DEFAULT 0 NOT NULL,
   PRIMARY KEY (upgrade_id),
   FOREIGN KEY (cluster_id) REFERENCES clusters(cluster_id),
   FOREIGN KEY (request_id) REFERENCES request(request_id)

ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql

@@ -795,6 +795,7 @@ CREATE TABLE upgrade (
   skip_failures BIT NOT NULL DEFAULT 0,
   skip_sc_failures BIT NOT NULL DEFAULT 0,
   downgrade_allowed BIT NOT NULL DEFAULT 1,
+  suspended BIT DEFAULT 0 NOT NULL,
   PRIMARY KEY CLUSTERED (upgrade_id),
   FOREIGN KEY (cluster_id) REFERENCES clusters(cluster_id),
   FOREIGN KEY (request_id) REFERENCES request(request_id)

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py

@@ -41,6 +41,7 @@ from resource_management.core.logger import Logger
 from utils import service, safe_zkfc_op, is_previous_fs_image
 from setup_ranger_hdfs import setup_ranger_hdfs, create_ranger_audit_hdfs_directories
 
+import namenode_upgrade
 
 def wait_for_safemode_off(hdfs_binary, afterwait_sleep=0, execute_kinit=False):
   """
@@ -77,7 +78,9 @@ def wait_for_safemode_off(hdfs_binary, afterwait_sleep=0, execute_kinit=False):
     Logger.error("NameNode is still in safemode, please be careful with commands that need safemode OFF.")
 
 @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
-def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None, env=None):
+def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None,
+    upgrade_suspended=False, env=None):
+
   if action is None:
     raise Fail('"action" parameter is required for function namenode().')
 
@@ -125,17 +128,23 @@ def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None, e
         options = "-rollingUpgrade started"
       elif params.upgrade_direction == Direction.DOWNGRADE:
         options = "-rollingUpgrade downgrade"
-        
     elif upgrade_type == "nonrolling":
       is_previous_image_dir = is_previous_fs_image()
-      Logger.info(format("Previous file system image dir present is {is_previous_image_dir}"))
+      Logger.info("Previous file system image dir present is {0}".format(str(is_previous_image_dir)))
 
       if params.upgrade_direction == Direction.UPGRADE:
         options = "-rollingUpgrade started"
       elif params.upgrade_direction == Direction.DOWNGRADE:
         options = "-rollingUpgrade downgrade"
+    elif upgrade_type is None and upgrade_suspended is True:
+      # the rollingUpgrade flag must be passed in during a suspended upgrade when starting NN
+      if os.path.exists(namenode_upgrade.get_upgrade_in_progress_marker()):
+        options = "-rollingUpgrade started"
+      else:
+        Logger.info("The NameNode upgrade marker file {0} does not exist, yet an upgrade is currently suspended. "
+                    "Assuming that the upgrade of NameNode has not occurred yet.".format(namenode_upgrade.get_upgrade_in_progress_marker()))
 
-    Logger.info(format("Option for start command: {options}"))
+    Logger.info("Options for start command are: {0}".format(options))
 
     service(
       action="start",
@@ -216,7 +225,9 @@ def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None, e
     decommission()
 
 @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
-def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None, env=None):
+def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None,
+    upgrade_suspended=False, env=None):
+
   if action is None:
     raise Fail('"action" parameter is required for function namenode().')
 

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py

@@ -99,7 +99,14 @@ class NameNode(Script):
     env.set_params(params)
     self.configure(env)
     hdfs_binary = self.get_hdfs_binary()
-    namenode(action="start", hdfs_binary=hdfs_binary, upgrade_type=upgrade_type, env=env)
+    namenode(action="start", hdfs_binary=hdfs_binary, upgrade_type=upgrade_type,
+      upgrade_suspended=params.upgrade_suspended, env=env)
+
+    # after starting NN in an upgrade, touch the marker file
+    if upgrade_type is not None:
+      # place a file on the system indicating that we've submitting the command that
+      # instructs NN that it is now part of an upgrade
+      namenode_upgrade.create_upgrade_marker()
 
   def stop(self, env, upgrade_type=None):
     import params

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode_upgrade.py

@@ -21,6 +21,7 @@ import os
 
 from resource_management.core.logger import Logger
 from resource_management.core.resources.system import Execute
+from resource_management.core.resources.system import File
 from resource_management.core import shell
 from resource_management.core.shell import as_user
 from resource_management.core.exceptions import Fail
@@ -35,6 +36,8 @@ from namenode_ha_state import NamenodeHAState
 safemode_to_instruction = {SafeMode.ON: "enter",
                            SafeMode.OFF: "leave"}
 
+NAMENODE_UPGRADE_IN_PROGRESS_MARKER_FILE = "namenode-upgrade-in-progress"
+
 def prepare_upgrade_check_for_previous_dir():
   """
   During a NonRolling (aka Express Upgrade), preparing the NameNode requires backing up some data.
@@ -268,4 +271,52 @@ def finalize_upgrade(upgrade_type, hdfs_binary):
           logoutput=True)
   Execute(query_cmd,
           user=params.hdfs_user,
-          logoutput=True)
+          logoutput=True)
+
+  # upgrade is finalized; remove the upgrade marker
+  delete_upgrade_marker()
+
+
+def get_upgrade_in_progress_marker():
+  """
+  Gets the full path of the file which indicates that NameNode has begun its stack upgrade.
+  :return:
+  """
+  from resource_management.libraries.script.script import Script
+  return os.path.join(Script.get_tmp_dir(), NAMENODE_UPGRADE_IN_PROGRESS_MARKER_FILE)
+
+
+def create_upgrade_marker():
+  """
+  Creates the marker file indicating that NameNode has begun participating in a stack upgrade.
+  If the file already exists, nothing will be done. This will silently log exceptions on failure.
+  :return:
+  """
+  # create the marker file which indicates
+  try:
+    namenode_upgrade_in_progress_marker = get_upgrade_in_progress_marker()
+    if not os.path.isfile(namenode_upgrade_in_progress_marker):
+      File(namenode_upgrade_in_progress_marker)
+  except:
+    Logger.warning("Unable to create NameNode upgrade marker file {0}".format(namenode_upgrade_in_progress_marker))
+
+
+def delete_upgrade_marker():
+  """
+  Removes the marker file indicating that NameNode has begun participating in a stack upgrade.
+  If the file does not exist, then nothing will be done.
+  Failure to remove this file could cause problems with restarts in the future. That's why
+  checking to see if there is a suspended upgrade is also advised. This function will raise
+  an exception if the file can't be removed.
+  :return:
+  """
+  # create the marker file which indicates
+  try:
+    namenode_upgrade_in_progress_marker = get_upgrade_in_progress_marker()
+    if os.path.isfile(namenode_upgrade_in_progress_marker):
+      File(namenode_upgrade_in_progress_marker, action='delete')
+  except:
+    error_message = "Unable to remove NameNode upgrade marker file {0}".format(namenode_upgrade_in_progress_marker)
+    Logger.error(error_message)
+    raise Fail(error_message)
+

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py

@@ -52,6 +52,9 @@ stack_version_formatted = format_stack_version(stack_version_unformatted)
 agent_stack_retry_on_unavailability = cbool(config["hostLevelParams"]["agent_stack_retry_on_unavailability"])
 agent_stack_retry_count = cint(config["hostLevelParams"]["agent_stack_retry_count"])
 
+# there is a stack upgrade which has not yet been finalized; it's currently suspended
+upgrade_suspended = default("roleParams/upgrade_suspended", False)
+
 # New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade
 version = default("/commandParams/version", None)
 

ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java

@@ -785,6 +785,7 @@ public class UpgradeResourceProviderTest {
     Map<String, Object> requestProps = new HashMap<String, Object>();
     requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_ID, id.toString());
     requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_STATUS, "ABORTED");
+    requestProps.put(UpgradeResourceProvider.UPGRADE_SUSPENDED, "false");
 
     UpgradeResourceProvider urp = createProvider(amc);
 
@@ -808,6 +809,7 @@ public class UpgradeResourceProviderTest {
     Map<String, Object> requestProps = new HashMap<String, Object>();
     requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_ID, id.toString());
     requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_STATUS, "ABORTED");
+    requestProps.put(UpgradeResourceProvider.UPGRADE_SUSPENDED, "true");
 
     UpgradeResourceProvider urp = createProvider(amc);
 
@@ -845,6 +847,25 @@ public class UpgradeResourceProviderTest {
     urp.updateResources(req, null);
   }
 
+  @Test(expected = IllegalArgumentException.class)
+  public void testAbortWithoutSuspendFlag() throws Exception {
+    RequestStatus status = testCreateResources();
+
+    Set<Resource> createdResources = status.getAssociatedResources();
+    assertEquals(1, createdResources.size());
+    Resource res = createdResources.iterator().next();
+    Long id = (Long) res.getPropertyValue("Upgrade/request_id");
+    assertNotNull(id);
+    assertEquals(Long.valueOf(1), id);
+
+    Map<String, Object> requestProps = new HashMap<String, Object>();
+    requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_ID, id.toString());
+    requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_STATUS, "ABORTED");
+
+    UpgradeResourceProvider urp = createProvider(amc);
+    Request req = PropertyHelper.getUpdateRequest(requestProps, null);
+    urp.updateResources(req, null);
+  }
 
   @Test
   public void testDirectionUpgrade() throws Exception {

ambari-web/app/controllers/main/admin/stack_and_upgrade_controller.js

@@ -581,6 +581,22 @@ App.MainAdminStackAndUpgradeController = Em.Controller.extend(App.LocalStorage,
       error: errorCallback
     });
   },
+  
+  /**
+   * suspend upgrade (in order to restart it later)
+   */
+  abortUpgradeWithSuspend: function () {
+    var errorCallback = this.get('isDowngrade') ? 'abortDowngradeErrorCallback' : 'abortUpgradeErrorCallback';
+    return App.ajax.send({
+      name: 'admin.upgrade.suspend',
+      sender: this,
+      data: {
+        upgradeId: this.get('upgradeId'),
+        isDowngrade: this.get('isDowngrade')
+      },
+      error: errorCallback
+    });
+  },  
 
   /**
    * error callback of <code>abortUpgrade()</code>
@@ -1681,7 +1697,7 @@ App.MainAdminStackAndUpgradeController = Em.Controller.extend(App.LocalStorage,
    */
   suspendUpgrade: function () {
     var self = this;
-    return this.abortUpgrade().done(function () {
+    return this.abortUpgradeWithSuspend().done(function () {
       App.set('upgradeState', 'ABORTED');
       self.setDBProperty('upgradeState', 'ABORTED');
       App.clusterStatus.setClusterStatus({

ambari-web/app/utils/ajax/ajax.js

@@ -1699,7 +1699,26 @@ var urls = {
             "downgrade": data.isDowngrade
           },
           "Upgrade": {
-            "request_status": "ABORTED"
+            "request_status": "ABORTED",
+            "suspended": "false"
+          }
+        })
+      }
+    }
+  },
+  'admin.upgrade.suspend': {
+    'real': '/clusters/{clusterName}/upgrades/{upgradeId}',
+    'mock': '',
+    'type': 'PUT',
+    'format': function (data) {
+      return {
+        data: JSON.stringify({
+          "RequestInfo": {
+            "downgrade": data.isDowngrade
+          },
+          "Upgrade": {
+            "request_status": "ABORTED",
+            "suspended": "true"
           }
         })
       }