AMBARI-14978. Create Upgrade Packs for Patch Upgrades (ncole)

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java

@@ -106,6 +106,7 @@ import org.apache.ambari.server.state.stack.upgrade.StageWrapper;
 import org.apache.ambari.server.state.stack.upgrade.Task;
 import org.apache.ambari.server.state.stack.upgrade.TaskWrapper;
 import org.apache.ambari.server.state.stack.upgrade.UpdateStackGrouping;
+import org.apache.ambari.server.state.stack.upgrade.UpgradeScope;
 import org.apache.ambari.server.state.stack.upgrade.UpgradeType;
 import org.apache.ambari.server.state.svccomphost.ServiceComponentHostServerActionEvent;
 import org.apache.commons.collections.CollectionUtils;
@@ -304,9 +305,9 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
         String forceDowngrade = requestInfoProps.get(UpgradeResourceDefinition.DOWNGRADE_DIRECTIVE);
         String clusterName = (String) requestMap.get(UPGRADE_CLUSTER_NAME);
 
-          if (null == clusterName) {
-              throw new AmbariException(String.format("%s is required", UPGRADE_CLUSTER_NAME));
-           }
+        if (null == clusterName) {
+          throw new AmbariException(String.format("%s is required", UPGRADE_CLUSTER_NAME));
+        }
 
         Cluster cluster = getManagementController().getClusters().getCluster(clusterName);
         Direction direction = Boolean.parseBoolean(forceDowngrade) ? Direction.DOWNGRADE
@@ -315,15 +316,17 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
         UpgradePack up = validateRequest(direction, requestMap);
 
         try {
-              return createUpgrade(cluster, direction, up, requestMap);
-            } catch (Exception e){
-              LOG.error("Error appears during upgrade task submitting", e);
-
-                // Any error caused in the createUpgrade will initiate transaction rollback
-                  // As we operate inside with cluster data, any cache which belongs to cluster need to be flushed
-                    cluster.invalidateData();
-              throw e;
-            }
+          return createUpgrade(cluster, direction, up, requestMap);
+        } catch (Exception e) {
+          LOG.error("Error appears during upgrade task submitting", e);
+
+          // Any error caused in the createUpgrade will initiate transaction
+          // rollback
+          // As we operate inside with cluster data, any cache which belongs to
+          // cluster need to be flushed
+          cluster.invalidateData();
+          throw e;
+        }
       }
     });
 
@@ -706,6 +709,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
     StackId targetStackId = null;
 
     Set<String> supportedServices = new HashSet<>();
+    UpgradeScope scope = UpgradeScope.COMPLETE;
 
     switch (direction) {
       case UPGRADE:
@@ -733,6 +737,10 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
             for (AvailableService available : services) {
               supportedServices.add(available.getName());
             }
+
+            if (!services.isEmpty()) {
+              scope = UpgradeScope.PARTIAL;
+            }
           }
         }
 
@@ -747,7 +755,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
     UpgradeContext ctx = new UpgradeContext(resolver, sourceStackId, targetStackId, version,
         direction, pack.getType());
     ctx.setSupportedServices(supportedServices);
-
+    ctx.setScope(scope);
 
     if (direction.isDowngrade()) {
       if (requestMap.containsKey(UPGRADE_FROM_VERSION)) {

ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java

@@ -27,6 +27,7 @@ import java.util.Set;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.stack.MasterHostResolver;
 import org.apache.ambari.server.state.stack.upgrade.Direction;
+import org.apache.ambari.server.state.stack.upgrade.UpgradeScope;
 import org.apache.ambari.server.state.stack.upgrade.UpgradeType;
 
 /**
@@ -88,6 +89,8 @@ public class UpgradeContext {
 
   private Set<String> m_supported = new HashSet<>();
 
+  private UpgradeScope m_scope = UpgradeScope.ANY;
+
   /**
    * Constructor.
    *
@@ -378,4 +381,12 @@ public class UpgradeContext {
 
     return false;
   }
+
+  public void setScope(UpgradeScope scope) {
+    m_scope = scope;
+  }
+
+  public boolean isScoped(UpgradeScope scope) {
+    return m_scope.isScoped(scope);
+  }
 }

ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java

@@ -270,6 +270,11 @@ public class UpgradeHelper {
 
     for (Grouping group : upgradePack.getGroups(context.getDirection())) {
 
+      // !!! grouping is not scoped to context
+      if (!context.isScoped(group.scope)) {
+        continue;
+      }
+
       UpgradeGroupHolder groupHolder = new UpgradeGroupHolder();
       groupHolder.name = group.name;
       groupHolder.title = group.title;

ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java

@@ -39,6 +39,7 @@ import org.apache.ambari.server.state.Host;
 import org.apache.ambari.server.state.MaintenanceState;
 import org.apache.ambari.server.state.UpgradeContext;
 import org.apache.ambari.server.state.stack.UpgradePack.ProcessingComponent;
+import org.apache.commons.lang.StringUtils;
 
 import com.google.gson.JsonArray;
 import com.google.gson.JsonObject;
@@ -96,6 +97,10 @@ public class ClusterGrouping extends Grouping {
 
     @XmlElement(name="task")
     public Task task;
+
+    @XmlElement(name="scope")
+    public UpgradeScope scope = UpgradeScope.ANY;
+
   }
 
   public class ClusterBuilder extends StageWrapperBuilder {
@@ -215,8 +220,18 @@ public class ClusterGrouping extends Grouping {
     String component = execution.component;
     ExecuteTask et = (ExecuteTask) execution.task;
 
-    if (null != service && !service.isEmpty() &&
-        null != component && !component.isEmpty()) {
+    if (StringUtils.isNotBlank(service) && StringUtils.isNotBlank(component)) {
+
+      // !!! if the context is not scoped for the execute-stage, bail
+      if (!ctx.isScoped(execution.scope)) {
+        return null;
+      }
+
+      // !!! if the context is targeted and does not include the service, bail
+      if (!ctx.isServiceSupported(service)) {
+        return null;
+      }
+
 
       HostsType hosts = ctx.getResolver().getMasterAndHosts(service, component);
 

ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ColocatedGrouping.java

@@ -80,11 +80,9 @@ public class ColocatedGrouping extends Grouping {
     }
 
     @Override
-    public void add(UpgradeContext ctx, HostsType hostsType, String service,
+    public void add(UpgradeContext context, HostsType hostsType, String service,
         boolean clientOnly, ProcessingComponent pc, Map<String, String> params) {
 
-      boolean forUpgrade = ctx.getDirection().isUpgrade();
-
       int count = Double.valueOf(Math.ceil(
           (double) m_batch.percent / 100 * hostsType.hosts.size())).intValue();
 
@@ -103,13 +101,13 @@ public class ColocatedGrouping extends Grouping {
 
         TaskProxy proxy = null;
 
-        List<Task> tasks = resolveTasks(forUpgrade, true, pc);
+        List<Task> tasks = resolveTasks(context, true, pc);
 
         if (null != tasks && tasks.size() > 0) {
           proxy = new TaskProxy();
           proxy.clientOnly = clientOnly;
           proxy.message = getStageText("Preparing",
-              ctx.getComponentDisplay(service, pc.name), Collections.singleton(host));
+              context.getComponentDisplay(service, pc.name), Collections.singleton(host));
           proxy.tasks.addAll(TaskWrapperBuilder.getTaskList(service, pc.name, singleHostsType, tasks, params));
           proxy.service = service;
           proxy.component = pc.name;
@@ -117,22 +115,20 @@ public class ColocatedGrouping extends Grouping {
         }
 
         // !!! FIXME upgrade definition have only one step, and it better be a restart
-        if (null != pc.tasks && 1 == pc.tasks.size()) {
-          Task t = pc.tasks.get(0);
-          if (RestartTask.class.isInstance(t)) {
-            proxy = new TaskProxy();
-            proxy.clientOnly = clientOnly;
-            proxy.tasks.add(new TaskWrapper(service, pc.name, Collections.singleton(host), params, t));
-            proxy.restart = true;
-            proxy.service = service;
-            proxy.component = pc.name;
-            proxy.message = getStageText("Restarting",
-                ctx.getComponentDisplay(service, pc.name), Collections.singleton(host));
-            targetList.add(proxy);
-          }
+        Task t = resolveTask(context, pc);
+        if (null != t && RestartTask.class.isInstance(t)) {
+          proxy = new TaskProxy();
+          proxy.clientOnly = clientOnly;
+          proxy.tasks.add(new TaskWrapper(service, pc.name, Collections.singleton(host), params, t));
+          proxy.restart = true;
+          proxy.service = service;
+          proxy.component = pc.name;
+          proxy.message = getStageText("Restarting",
+              context.getComponentDisplay(service, pc.name), Collections.singleton(host));
+          targetList.add(proxy);
         }
 
-        tasks = resolveTasks(forUpgrade, false, pc);
+        tasks = resolveTasks(context, false, pc);
 
         if (null != tasks && tasks.size() > 0) {
           proxy = new TaskProxy();
@@ -141,7 +137,7 @@ public class ColocatedGrouping extends Grouping {
           proxy.service = service;
           proxy.tasks.addAll(TaskWrapperBuilder.getTaskList(service, pc.name, singleHostsType, tasks, params));
           proxy.message = getStageText("Completing",
-              ctx.getComponentDisplay(service, pc.name), Collections.singleton(host));
+              context.getComponentDisplay(service, pc.name), Collections.singleton(host));
           targetList.add(proxy);
         }
       }

ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java

@@ -17,16 +17,6 @@
  */
 package org.apache.ambari.server.state.stack.upgrade;
 
-import org.apache.ambari.server.stack.HostsType;
-import org.apache.ambari.server.state.UpgradeContext;
-import org.apache.ambari.server.state.stack.UpgradePack;
-import org.apache.ambari.server.state.stack.UpgradePack.ProcessingComponent;
-import org.apache.ambari.server.utils.SetUtils;
-import org.apache.commons.lang.StringUtils;
-
-import javax.xml.bind.annotation.XmlAttribute;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlSeeAlso;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -34,6 +24,17 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import javax.xml.bind.annotation.XmlAttribute;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlSeeAlso;
+
+import org.apache.ambari.server.stack.HostsType;
+import org.apache.ambari.server.state.UpgradeContext;
+import org.apache.ambari.server.state.stack.UpgradePack;
+import org.apache.ambari.server.state.stack.UpgradePack.ProcessingComponent;
+import org.apache.ambari.server.utils.SetUtils;
+import org.apache.commons.lang.StringUtils;
+
 /**
  *
  */
@@ -67,6 +68,9 @@ public class Grouping {
   @XmlElement(name="parallel-scheduler")
   public ParallelScheduler parallelScheduler;
 
+  @XmlElement(name="scope")
+  public UpgradeScope scope = UpgradeScope.ANY;
+
   /**
    * Gets the default builder.
    */
@@ -88,44 +92,43 @@ public class Grouping {
     /**
      * Add stages where the restart stages are ordered
      * E.g., preupgrade, restart hosts(0), ..., restart hosts(n-1), postupgrade
+     * @param context the context
      * @param hostsType the order collection of hosts, which may have a master and secondary
      * @param service the service name
      * @param pc the ProcessingComponent derived from the upgrade pack.
      * @param params additional parameters
      */
     @Override
-    public void add(UpgradeContext ctx, HostsType hostsType, String service,
+    public void add(UpgradeContext context, HostsType hostsType, String service,
        boolean clientOnly, ProcessingComponent pc, Map<String, String> params) {
 
-      boolean forUpgrade = ctx.getDirection().isUpgrade();
-
       // Construct the pre tasks during Upgrade/Downgrade direction.
       // Buckets are grouped by the type, e.g., bucket of all Execute tasks, or all Configure tasks.
-      List<TaskBucket> buckets = buckets(resolveTasks(forUpgrade, true, pc));
+      List<TaskBucket> buckets = buckets(resolveTasks(context, true, pc));
       for (TaskBucket bucket : buckets) {
         // The TaskWrappers take into account if a task is meant to run on all, any, or master.
         // A TaskWrapper may contain multiple tasks, but typically only one, and they all run on the same set of hosts.
         List<TaskWrapper> preTasks = TaskWrapperBuilder.getTaskList(service, pc.name, hostsType, bucket.tasks, params);
         List<List<TaskWrapper>> organizedTasks = organizeTaskWrappersBySyncRules(preTasks);
         for (List<TaskWrapper> tasks : organizedTasks) {
-          addTasksToStageInBatches(tasks, "Preparing", ctx, service, pc, params);
+          addTasksToStageInBatches(tasks, "Preparing", context, service, pc, params);
         }
       }
 
       // Add the processing component
-      if (null != pc.tasks && 1 == pc.tasks.size()) {
-        Task t = pc.tasks.get(0);
+      Task t = resolveTask(context, pc);
+      if (null != t) {
         TaskWrapper tw = new TaskWrapper(service, pc.name, hostsType.hosts, params, Collections.singletonList(t));
-        addTasksToStageInBatches(Collections.singletonList(tw), t.getActionVerb(), ctx, service, pc, params);
+        addTasksToStageInBatches(Collections.singletonList(tw), t.getActionVerb(), context, service, pc, params);
       }
 
       // Construct the post tasks during Upgrade/Downgrade direction.
-      buckets = buckets(resolveTasks(forUpgrade, false, pc));
+      buckets = buckets(resolveTasks(context, false, pc));
       for (TaskBucket bucket : buckets) {
         List<TaskWrapper> postTasks = TaskWrapperBuilder.getTaskList(service, pc.name, hostsType, bucket.tasks, params);
         List<List<TaskWrapper>> organizedTasks = organizeTaskWrappersBySyncRules(postTasks);
         for (List<TaskWrapper> tasks : organizedTasks) {
-          addTasksToStageInBatches(tasks, "Completing", ctx, service, pc, params);
+          addTasksToStageInBatches(tasks, "Completing", context, service, pc, params);
         }
       }
 
@@ -157,7 +160,7 @@ public class Grouping {
           subTasks.add(tw);
         }
       }
-      
+
       if (!subTasks.isEmpty()) {
         groupedTasks.add(subTasks);
       }
@@ -208,7 +211,7 @@ public class Grouping {
         }
       }
     }
-    
+
     /**
      * Determine if service checks need to be ran after the stages.
      * @param upgradeContext the upgrade context

ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java

@@ -195,23 +195,56 @@ public abstract class StageWrapperBuilder {
    *   <li>When performing a downgrade, use the downgrade tasks if they are defined</li>
    *   <li>When performing a downgrade, but no downgrade tasks exist, reuse the upgrade tasks</li>
    * </ul>
-   * @param forUpgrade  {@code true} if resolving for an upgrade, {@code false} for downgrade
+   * @param context     the upgrade context
    * @param preTasks    {@code true} if loading pre-upgrade or pre-downgrade
    * @param pc          the processing component holding task definitions
    * @return A collection, potentially empty, of the tasks to run, which may contain either
    * pre or post tasks if they exist, and the order depends on whether it's an upgrade or downgrade.
    */
-  protected List<Task> resolveTasks(boolean forUpgrade, boolean preTasks, ProcessingComponent pc) {
+  protected List<Task> resolveTasks(final UpgradeContext context, boolean preTasks, ProcessingComponent pc) {
     if (null == pc) {
       return Collections.emptyList();
     }
 
+    boolean forUpgrade = context.getDirection().isUpgrade();
+
+    final List<Task> interim;
+
     if (forUpgrade) {
-      return preTasks ? pc.preTasks : pc.postTasks;
+      interim = preTasks ? pc.preTasks : pc.postTasks;
     } else {
-      return preTasks ?
+      interim = preTasks ?
         (null == pc.preDowngradeTasks ? pc.preTasks : pc.preDowngradeTasks) :
         (null == pc.postDowngradeTasks ? pc.postTasks : pc.postDowngradeTasks);
     }
+
+    if (null == interim || interim.isEmpty()) {
+      return Collections.emptyList();
+    }
+
+    List<Task> tasks = new ArrayList<>();
+    for (Task t : interim) {
+      if (context.isScoped(t.scope)) {
+        tasks.add(t);
+      }
+    }
+
+    return tasks;
+  }
+
+  /**
+   * The upgrade packs are written such that there is one and only one upgrade element
+   * for a component, all other directives go in (pre|post)-(upgrade|downgrade) elements.
+   * @param pc the processing component
+   * @return the single task, or {@code null} if there is none
+   */
+  protected Task resolveTask(UpgradeContext context, ProcessingComponent pc) {
+    if (null != pc.tasks && 1 == pc.tasks.size()) {
+      if (context.isScoped(pc.tasks.get(0).scope)) {
+        return pc.tasks.get(0);
+      }
+    }
+
+    return null;
   }
 }

ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java

@@ -56,6 +56,13 @@ public abstract class Task {
    */
   public abstract String getActionVerb();
 
+  /**
+   * The scope for the task
+   */
+  @XmlElement(name = "scope")
+  public UpgradeScope scope = UpgradeScope.ANY;
+
+
   @Override
   public String toString() {
     return getType().toString();

ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/UpgradeScope.java

@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.state.stack.upgrade;
+
+import javax.xml.bind.annotation.XmlEnum;
+import javax.xml.bind.annotation.XmlEnumValue;
+
+import com.google.gson.annotations.SerializedName;
+
+/**
+ * Indicates the scope of a group or task
+ */
+@XmlEnum
+public enum UpgradeScope {
+
+  /**
+   * Used only when completely upgrading the cluster.
+   */
+  @XmlEnumValue("COMPLETE")
+  @SerializedName("rolling_upgrade")
+  COMPLETE,
+
+  /**
+   * Used only when partially upgrading the cluster.
+   */
+  @XmlEnumValue("PARTIAL")
+  @SerializedName("partial")
+  PARTIAL,
+
+  /**
+   * Used for any scoped upgrade.
+   */
+  @XmlEnumValue("ANY")
+  @SerializedName("any")
+  ANY;
+
+  public boolean isScoped(UpgradeScope scope) {
+    if (ANY == this || ANY == scope) {
+      return true;
+    }
+
+    return this == scope;
+  }
+
+}

ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml

@@ -524,6 +524,7 @@
     that are installed on the hosts but not known by Ambari.
     -->
     <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
+      <scope>COMPLETE</scope>
       <skippable>true</skippable>
       <supports-auto-skip-failure>false</supports-auto-skip-failure>
 

ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml

@@ -332,6 +332,7 @@
     </group>
 
     <group xsi:type="cluster" name="ALL_HOST_OPS" title="Finalize Hosts">
+      <scope>COMPLETE</scope>
       <execute-stage title="Update remaining HDP stack to {{version}}">
         <task xsi:type="execute">
           <script>scripts/ru_set_all.py</script>

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml

@@ -605,6 +605,7 @@
     that are installed on the hosts but not known by Ambari.
     -->
     <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
+      <scope>COMPLETE</scope>
       <skippable>true</skippable>
       <supports-auto-skip-failure>false</supports-auto-skip-failure>
 

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml

@@ -393,6 +393,7 @@
     </group>
 
     <group xsi:type="cluster" name="ALL_HOST_OPS" title="Finalize Hosts">
+      <scope>COMPLETE</scope>
       <execute-stage title="Update remaining HDP stack to {{version}}">
         <task xsi:type="execute">
           <script>scripts/ru_set_all.py</script>

ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml

@@ -600,6 +600,7 @@
     that are installed on the hosts but not known by Ambari.
     -->
     <group xsi:type="cluster" name="ALL_HOST_OPS" title="Set Version On All Hosts">
+      <scope>COMPLETE</scope>
       <skippable>true</skippable>
       <supports-auto-skip-failure>false</supports-auto-skip-failure>
 

ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml

@@ -391,6 +391,7 @@
     </group>
 
     <group xsi:type="cluster" name="ALL_HOST_OPS" title="Finalize Hosts">
+      <scope>COMPLETE</scope>
       <execute-stage title="Update remaining HDP stack to {{version}}">
         <task xsi:type="execute">
           <script>scripts/ru_set_all.py</script>

ambari-server/src/main/resources/version_definition.xsd

@@ -54,6 +54,7 @@
       <xs:enumeration value="ubuntu12" />
       <xs:enumeration value="ubuntu14" />
       <xs:enumeration value="suse11" />
+      <xs:enumeration value="amazon2015" />
     </xs:restriction>
   </xs:simpleType>
   

ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java

@@ -36,9 +36,6 @@ import static org.junit.Assert.fail;
 
 import java.io.StringReader;
 import java.lang.reflect.Type;
-import java.net.ConnectException;
-import java.net.MalformedURLException;
-import java.net.UnknownHostException;
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -107,7 +104,6 @@ import org.apache.ambari.server.orm.entities.WidgetLayoutUserWidgetEntity;
 import org.apache.ambari.server.security.TestAuthenticationFactory;
 import org.apache.ambari.server.security.authorization.AuthorizationException;
 import org.apache.ambari.server.security.authorization.Users;
-import org.apache.ambari.server.security.authorization.internal.InternalAuthenticationToken;
 import org.apache.ambari.server.serveraction.ServerAction;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
@@ -156,6 +152,7 @@ import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.security.core.context.SecurityContextHolder;
 
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
@@ -165,7 +162,6 @@ import com.google.inject.Injector;
 import com.google.inject.persist.PersistService;
 
 import junit.framework.Assert;
-import org.springframework.security.core.context.SecurityContextHolder;
 
 public class AmbariManagementControllerTest {
 
@@ -7183,7 +7179,7 @@ public class AmbariManagementControllerTest {
     Assert.assertEquals(1, responsesWithParams.size());
     StackVersionResponse resp = responsesWithParams.iterator().next();
     assertNotNull(resp.getUpgradePacks());
-    assertEquals(8, resp.getUpgradePacks().size());
+    assertEquals(9, resp.getUpgradePacks().size());
     assertTrue(resp.getUpgradePacks().contains("upgrade_test"));
   }
 

ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java

@@ -60,6 +60,7 @@ import org.apache.ambari.server.state.stack.upgrade.ManualTask;
 import org.apache.ambari.server.state.stack.upgrade.StageWrapper;
 import org.apache.ambari.server.state.stack.upgrade.Task;
 import org.apache.ambari.server.state.stack.upgrade.TaskWrapper;
+import org.apache.ambari.server.state.stack.upgrade.UpgradeScope;
 import org.apache.ambari.server.state.stack.upgrade.UpgradeType;
 import org.apache.ambari.server.utils.EventBusSynchronizer;
 import org.easymock.EasyMock;
@@ -133,26 +134,6 @@ public class UpgradeHelperTest {
     m_masterHostResolver = EasyMock.createMock(MasterHostResolver.class);
     m_managementController = injector.getInstance(AmbariManagementController.class);
 
-//    StackDAO stackDAO = injector.getInstance(StackDAO.class);
-//    StackEntity stackEntity = new StackEntity();
-//    stackEntity.setStackName("HDP");
-//    stackEntity.setStackVersion("2.1");
-//    stackDAO.create(stackEntity);
-//
-//    StackEntity stackEntityTo = new StackEntity();
-//    stackEntityTo.setStackName("HDP");
-//    stackEntityTo.setStackVersion("2.2");
-//    stackDAO.create(stackEntityTo);
-//
-//    Clusters clusters = injector.getInstance(Clusters.class);
-//    clusters.addCluster("c1", new StackId("HDP", "2.1"));
-//
-//    RepositoryVersionDAO repositoryVersionDAO = injector.getInstance(RepositoryVersionDAO.class);
-//    repositoryVersionDAO.create(stackEntity, "2.1.1", "2.1.1", "");
-//    repositoryVersionDAO.create(stackEntityTo, "2.2.0", "2.2.0", "");
-//
-//    replay(m_configHelper);
-
     // Set the authenticated user
     // TODO: remove this or replace the authenticated user to test authorization rules
     SecurityContextHolder.getContext().setAuthentication(TestAuthenticationFactory.createAdministrator("admin"));
@@ -260,7 +241,7 @@ public class UpgradeHelperTest {
   }
 
   @Test
-  public void testSupportedServiceUpgradeOrchestration() throws Exception {
+  public void testPartialUpgradeOrchestration() throws Exception {
     Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("foo", "bar");
     assertTrue(upgrades.isEmpty());
 
@@ -271,8 +252,8 @@ public class UpgradeHelperTest {
     ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER");
     ci.setDisplayName("ZooKeeper1 Server2");
 
-    assertTrue(upgrades.containsKey("upgrade_test"));
-    UpgradePack upgrade = upgrades.get("upgrade_test");
+    assertTrue(upgrades.containsKey("upgrade_test_partial"));
+    UpgradePack upgrade = upgrades.get("upgrade_test_partial");
     assertNotNull(upgrade);
 
     makeCluster();
@@ -280,7 +261,11 @@ public class UpgradeHelperTest {
     UpgradeContext context = new UpgradeContext(m_masterHostResolver, HDP_21,
         HDP_21, UPGRADE_VERSION, Direction.UPGRADE, UpgradeType.ROLLING);
     context.setSupportedServices(Collections.singleton("ZOOKEEPER"));
+    context.setScope(UpgradeScope.PARTIAL);
 
+    List<Grouping> groupings = upgrade.getGroups(Direction.UPGRADE);
+    assertEquals(8, groupings.size());
+    assertEquals(UpgradeScope.COMPLETE, groupings.get(6).scope);
 
     List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
 
@@ -290,26 +275,77 @@ public class UpgradeHelperTest {
     assertEquals("ZOOKEEPER", groups.get(1).name);
     assertEquals("POST_CLUSTER", groups.get(2).name);
 
-
     UpgradeGroupHolder group = groups.get(1);
     // check that the display name is being used
     assertTrue(group.items.get(1).getText().contains("ZooKeeper1 Server2"));
-    assertEquals(group.items.get(5).getText(), "Service Check Zk");
+    assertEquals("Service Check Zk", group.items.get(6).getText());
 
     UpgradeGroupHolder postGroup = groups.get(2);
     assertEquals("POST_CLUSTER", postGroup.name);
     assertEquals("Finalize Upgrade", postGroup.title);
-    assertEquals(3, postGroup.items.size());
+    assertEquals(2, postGroup.items.size());
     assertEquals("Confirm Finalize", postGroup.items.get(0).getText());
-    assertEquals("Execute HDFS Finalize", postGroup.items.get(1).getText());
-    assertEquals("Save Cluster State", postGroup.items.get(2).getText());
-    assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType());
+    assertEquals("Save Cluster State", postGroup.items.get(1).getText());
+    assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(1).getType());
 
-    assertEquals(4, groups.get(0).items.size());
-    assertEquals(6, groups.get(1).items.size());
-    assertEquals(3, groups.get(2).items.size());
+    assertEquals(3, groups.get(0).items.size());
+    assertEquals(7, groups.get(1).items.size());
+    assertEquals(2, groups.get(2).items.size());
   }
 
+  @Test
+  public void testCompleteUpgradeOrchestration() throws Exception {
+    Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("foo", "bar");
+    assertTrue(upgrades.isEmpty());
+
+    upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1");
+
+    ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER");
+    si.setDisplayName("Zk");
+    ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER");
+    ci.setDisplayName("ZooKeeper1 Server2");
+
+    assertTrue(upgrades.containsKey("upgrade_test_partial"));
+    UpgradePack upgrade = upgrades.get("upgrade_test_partial");
+    assertNotNull(upgrade);
+
+    makeCluster();
+
+    UpgradeContext context = new UpgradeContext(m_masterHostResolver, HDP_21,
+        HDP_21, UPGRADE_VERSION, Direction.UPGRADE, UpgradeType.ROLLING);
+    context.setSupportedServices(Collections.singleton("ZOOKEEPER"));
+    context.setScope(UpgradeScope.COMPLETE);
+
+    List<Grouping> groupings = upgrade.getGroups(Direction.UPGRADE);
+    assertEquals(8, groupings.size());
+    assertEquals(UpgradeScope.COMPLETE, groupings.get(6).scope);
+
+    List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
+
+    assertEquals(4, groups.size());
+
+    assertEquals("PRE_CLUSTER", groups.get(0).name);
+    assertEquals("ZOOKEEPER", groups.get(1).name);
+    assertEquals("ALL_HOSTS", groups.get(2).name);
+    assertEquals("POST_CLUSTER", groups.get(3).name);
+
+    UpgradeGroupHolder group = groups.get(1);
+    // check that the display name is being used
+    assertTrue(group.items.get(1).getText().contains("ZooKeeper1 Server2"));
+    assertEquals("Service Check Zk", group.items.get(5).getText());
+
+    UpgradeGroupHolder postGroup = groups.get(3);
+    assertEquals("POST_CLUSTER", postGroup.name);
+    assertEquals("Finalize Upgrade", postGroup.title);
+    assertEquals(2, postGroup.items.size());
+    assertEquals("Confirm Finalize", postGroup.items.get(0).getText());
+    assertEquals("Save Cluster State", postGroup.items.get(1).getText());
+    assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(1).getType());
+
+    assertEquals(3, groups.get(0).items.size());
+    assertEquals(6, groups.get(1).items.size());
+    assertEquals(1, groups.get(2).items.size());
+  }
 
   @Test
   public void testUpgradeServerActionOrchestration() throws Exception {

ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_partial.xml

@@ -0,0 +1,259 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<upgrade xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <target>2.2.*.*</target>
+  <target-stack>HDP-2.2.0</target-stack>
+  <type>ROLLING</type>
+
+  <prerequisite-checks>
+    <!-- List of additional pre-req checks to run in addition to the required pre-reqs -->
+    <check>org.apache.ambari.server.checks.HiveMultipleMetastoreCheck</check>
+    <check>org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck</check>
+    <check>org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck</check>
+    <check>org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck</check>
+    <check>org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck</check>
+    <check>org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck</check>
+    <check>org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck</check>
+    <check>org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck</check>
+    <check>org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck</check>
+  </prerequisite-checks>
+  
+  <order>
+    <group xsi:type="cluster" name="PRE_CLUSTER" title="Pre {{direction.text.proper}}">
+      <execute-stage title="Confirm 1">
+        <task xsi:type="manual">
+          <message>Foo</message>
+        </task>
+      </execute-stage>
+      <execute-stage service="HDFS" component="NAMENODE" title="Pre Upgrade HIVE">
+        <task xsi:type="manual">
+          <message>Back stuff up.</message>
+        </task>
+      </execute-stage>
+      <execute-stage service="HDFS" component="NAMENODE" title="Finalize HDFS">
+        <task xsi:type="execute">
+          <command>ls</command>
+        </task>
+      </execute-stage>
+      <execute-stage title="Confirm 2">
+        <task xsi:type="manual">
+          <message>Foo</message>
+        </task>
+      </execute-stage>
+    </group>
+  
+    <group name="ZOOKEEPER" title="Zookeeper">
+      <skippable>true</skippable>
+      <allow-retry>false</allow-retry>
+      <service name="ZOOKEEPER">
+        <component>ZOOKEEPER_SERVER</component>
+        <component>ZOOKEEPER_CLIENT</component>
+      </service>
+    </group>
+    
+    <group name="CORE_MASTER" title="Core Masters">
+      <service name="HDFS">
+        <component>JOURNALNODE</component>
+        <component>NAMENODE</component>
+      </service>
+      <service name="YARN">
+        <component>RESOURCEMANAGER</component>
+      </service>
+    </group>
+    
+    <group name="CORE_SLAVES" title="Core Slaves" xsi:type="colocated">
+      <skippable>true</skippable>      <!-- set skippable for test -->
+      <allow-retry>false</allow-retry> <!-- set no retry for test -->
+      <service name="HDFS">
+        <component>DATANODE</component>
+      </service>
+      <service name="HBASE">
+        <component>REGIONSERVER</component>
+      </service>
+      <service name="YARN">
+        <component>NODEMANAGER</component>
+      </service>
+      
+      <batch>
+        <percent>20</percent>
+        <message>Please run additional tests on {{components}}</message>
+      </batch>
+    </group>
+    
+    <group name="HIVE" title="Hive">
+      <skippable>true</skippable>
+      <service name="HIVE">
+        <component>HIVE_METASTORE</component>
+        <component>HIVE_SERVER</component>
+        <component>WEBHCAT_SERVER</component>
+      </service>
+    </group>
+
+    <group name="OOZIE" title="Oozie">
+      <skippable>true</skippable>
+      <supports-auto-skip-failure>false</supports-auto-skip-failure>
+      <service-check>false</service-check>
+      <service name="OOZIE">
+        <component>OOZIE_SERVER</component>
+        <component>OOZIE_CLIENT</component>
+      </service>
+    </group>
+    
+    <group xsi:type="cluster" name="ALL_HOSTS" title="All Hosts">
+      <scope>COMPLETE</scope>
+      <execute-stage title="All Hosts">
+        <task xsi:type="manual">
+          <message>Something to run on all hosts</message>
+        </task>
+      </execute-stage>
+    </group>
+    
+    <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
+      <execute-stage title="Confirm Finalize">
+        <task xsi:type="manual">
+          <message>Please confirm you are ready to finalize</message>
+        </task>
+      </execute-stage>
+      <execute-stage service="HDFS" component="NAMENODE" title="Execute HDFS Finalize">
+        <task xsi:type="execute">
+          <command>ls</command>
+        </task>
+      </execute-stage>
+      <execute-stage title="Save Cluster State" service="" component="">
+        <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FinalizeUpgradeAction">
+        </task>
+      </execute-stage>
+    </group>
+  </order>
+
+  <processing>
+    <service name="ZOOKEEPER">
+      <component name="ZOOKEEPER_SERVER">
+        <pre-upgrade>
+          <task xsi:type="manual">
+            <summary>SUMMARY OF PREPARE</summary>
+            <message>This is a manual task with a placeholder of {{foo/bar}}</message>
+          </task>
+          <task xsi:type="manual">
+            <scope>PARTIAL</scope>
+            <summary>Only partials</summary>
+            <message>This is only for partials</message>
+          </task>
+        </pre-upgrade>
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+        <post-upgrade>
+          <task xsi:type="configure" id="2.2.0" />
+        </post-upgrade>
+      </component>
+    </service>
+    
+    <service name="HDFS">
+      <component name="NAMENODE">
+        <pre-upgrade>
+          <task xsi:type="execute" hosts="master">
+            <command>su - {hdfs-user} -c 'dosomething'</command>
+          </task>
+          <task xsi:type="configure" id="hdp_2_1_1_nn_pre_upgrade" />
+          <task xsi:type="manual">
+            <message>{{direction.verb.proper}} your database</message>
+          </task>
+        </pre-upgrade>
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+        <post-upgrade>
+          <task xsi:type="execute">
+            <command>ls</command>
+          </task>
+        </post-upgrade>
+      </component>
+      <component name="DATANODE">
+        <pre-downgrade />
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+        <post-downgrade>
+          <task xsi:type="manual">
+            <message>Manual Downgrade</message>
+          </task>
+        </post-downgrade>
+      </component>
+    </service>
+    
+    <service name="YARN">
+      <component name="RESOURCEMANAGER">
+        <pre-upgrade>
+          <task xsi:type="execute">
+            <command>ls</command>
+          </task>
+        </pre-upgrade>
+      </component>
+      <component name="NODEMANAGER">
+        <pre-upgrade>
+          <task xsi:type="execute">
+            <command>ls</command>
+          </task>
+          <task xsi:type="configure" id="hdp_2_1_1_nm_pre_upgrade"/>
+        </pre-upgrade>
+      </component>
+    </service>
+    
+    <service name="HIVE">
+      <component name="HIVE_SERVER">
+        <pre-upgrade>
+          <task xsi:type="manual">
+            <summary>HiveServer Port Availability</summary>
+            <message>The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message>
+          </task>
+          <task xsi:type="configure" id="hdp_2_1_1_set_transport_mode"/>
+          <task xsi:type="configure" id="hdp_2_1_1_hive_server_foo"/>
+        </pre-upgrade>
+       </component>
+     </service>
+
+    <service name="OOZIE">
+      <component name="OOZIE_SERVER">
+        <pre-upgrade>
+          <!-- This is important, do not remove it since UpgradeHelperTest.java :
+          testUpgradeWithMultipleTasksWithMultipleHostTypes() asserts
+          that these tasks each run on their own stage. -->
+          <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
+            <script>scripts/oozie_server.py</script>
+            <function>stop</function>
+          </task>
+
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
+            <script>scripts/oozie_server_upgrade.py</script>
+            <function>upgrade_oozie_database_and_sharelib</function>
+          </task>
+        </pre-upgrade>
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+      </component>
+
+      <component name="OOZIE_CLIENT">
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+      </component>
+    </service>
+  </processing>
+</upgrade>