AMBARI-14924. Create patch upgrade orchestration based on available services (ncole)

ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java

@@ -53,6 +53,7 @@ public class HostsRepositoryVersionCheck extends AbstractCheckDescriptor {
     super(CheckDescription.HOSTS_REPOSITORY_VERSION);
   }
 
+  @Override
   public boolean isApplicable(PrereqCheckRequest request) throws AmbariException {
     return super.isApplicable(request) && request.getRepositoryVersion() != null;
   }
@@ -77,7 +78,7 @@ public class HostsRepositoryVersionCheck extends AbstractCheckDescriptor {
         for (HostVersionEntity hve : hostVersionDaoProvider.get().findByHost(host.getHostName())) {
 
           if (hve.getRepositoryVersion().getVersion().equals(request.getRepositoryVersion())
-              && hve.getState() == RepositoryVersionState.INSTALLED) {
+              && (hve.getState() == RepositoryVersionState.INSTALLED || hve.getState() == RepositoryVersionState.NOT_REQUIRED)) {
             found = true;
             break;
           }

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java

@@ -24,6 +24,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.VERSION;
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -36,8 +37,6 @@ import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import com.google.inject.assistedinject.Assisted;
-import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.Role;
 import org.apache.ambari.server.RoleCommand;
@@ -86,12 +85,15 @@ import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Config;
 import org.apache.ambari.server.state.ConfigHelper;
 import org.apache.ambari.server.state.DesiredConfig;
+import org.apache.ambari.server.state.RepositoryType;
 import org.apache.ambari.server.state.ServiceInfo;
 import org.apache.ambari.server.state.StackId;
 import org.apache.ambari.server.state.StackInfo;
 import org.apache.ambari.server.state.UpgradeContext;
 import org.apache.ambari.server.state.UpgradeHelper;
 import org.apache.ambari.server.state.UpgradeHelper.UpgradeGroupHolder;
+import org.apache.ambari.server.state.repository.AvailableService;
+import org.apache.ambari.server.state.repository.VersionDefinitionXml;
 import org.apache.ambari.server.state.stack.ConfigUpgradePack;
 import org.apache.ambari.server.state.stack.PrereqCheckStatus;
 import org.apache.ambari.server.state.stack.UpgradePack;
@@ -115,6 +117,8 @@ import com.google.common.collect.Lists;
 import com.google.gson.Gson;
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.assistedinject.Assisted;
+import com.google.inject.persist.Transactional;
 
 /**
  * Manages the ability to start and get status of upgrades.
@@ -701,12 +705,37 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
     StackId sourceStackId = null;
     StackId targetStackId = null;
 
+    Set<String> supportedServices = new HashSet<>();
+
     switch (direction) {
       case UPGRADE:
         sourceStackId = cluster.getCurrentStackVersion();
 
         RepositoryVersionEntity targetRepositoryVersion = s_repoVersionDAO.findByStackNameAndVersion(
             sourceStackId.getStackName(), version);
+
+        EnumSet<RepositoryType> serviceAware = EnumSet.of(RepositoryType.PATCH, RepositoryType.SERVICE);
+        if (serviceAware.contains(targetRepositoryVersion.getType())) {
+
+          VersionDefinitionXml xml = null;
+          StackInfo stackInfo = s_metaProvider.get().getStack(sourceStackId.getStackName(),
+              sourceStackId.getStackVersion());
+
+          try {
+            xml = targetRepositoryVersion.getRepositoryXml();
+          } catch (Exception e) {
+            throw new AmbariException(String.format("Could not load repository definition for version %s", version));
+          }
+
+          if (null != xml) {
+            Collection<AvailableService> services = xml.getAvailableServices(stackInfo);
+
+            for (AvailableService available : services) {
+              supportedServices.add(available.getName());
+            }
+          }
+        }
+
         targetStackId = targetRepositoryVersion.getStackId();
         break;
       case DOWNGRADE:
@@ -717,6 +746,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
 
     UpgradeContext ctx = new UpgradeContext(resolver, sourceStackId, targetStackId, version,
         direction, pack.getType());
+    ctx.setSupportedServices(supportedServices);
+
 
     if (direction.isDowngrade()) {
       if (requestMap.containsKey(UPGRADE_FROM_VERSION)) {

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java

@@ -262,7 +262,7 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc
 
     List<RepositoryVersionEntity> entities = s_repoVersionDAO.findByStack(entity.getStackId());
     if (entities.isEmpty()) {
-      throw new AmbariException(String.format("Patch %s was uploaded, but there are no repositories for %s",
+      throw new IllegalArgumentException(String.format("Patch %s was uploaded, but there are no repositories for %s",
           entity.getVersion(), entity.getStackId().toString()));
     }
 
@@ -290,7 +290,7 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc
     if (matching.isEmpty()) {
       String format = "No versions matched pattern %s";
 
-      throw new AmbariException(String.format(format,
+      throw new IllegalArgumentException(String.format(format,
           emptyCompatible ? holder.xml.release.version : holder.xml.release.compatibleWith));
     } else if (matching.size() > 1) {
       Set<String> versions= new HashSet<>();
@@ -298,7 +298,7 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc
         versions.add(match.getVersion());
       }
 
-      throw new AmbariException(String.format("More than one repository matches patch %s: %s",
+      throw new IllegalArgumentException(String.format("More than one repository matches patch %s: %s",
           entity.getVersion(), StringUtils.join(versions, ", ")));
     }
 

ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java

@@ -349,7 +349,7 @@ public class RepositoryVersionEntity {
   /**
    * Parse the version XML into its object representation.  This causes the XML to be lazy-loaded
    * from storage.
-   * @return {@code null} if the XSD is not available.
+   * @return {@code null} if the XSD (from the XML) is not available.
    * @throws Exception
    */
   public VersionDefinitionXml getRepositoryXml() throws Exception {

ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java

@@ -19,8 +19,10 @@ package org.apache.ambari.server.state;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.stack.MasterHostResolver;
@@ -84,6 +86,8 @@ public class UpgradeContext {
    */
   private boolean m_autoSkipManualVerification = false;
 
+  private Set<String> m_supported = new HashSet<>();
+
   /**
    * Constructor.
    *
@@ -350,4 +354,28 @@ public class UpgradeContext {
   public void setAutoSkipManualVerification(boolean autoSkipManualVerification) {
     m_autoSkipManualVerification = autoSkipManualVerification;
   }
+
+  /**
+   * Sets the service names that are supported by an upgrade.  This is used for
+   * {@link RepositoryType#PATCH} and {@link RepositoryType#SERVICE}.
+   *
+   * @param services  the set of specific services
+   */
+  public void setSupportedServices(Set<String> services) {
+    m_supported = services;
+  }
+
+  /**
+   * Gets if a service is supported.  If there are no services marked for the context,
+   * then ALL services are supported
+   * @param serviceName the service name to check.
+   * @return {@code true} when the service is supported
+   */
+  public boolean isServiceSupported(String serviceName) {
+    if (m_supported.isEmpty() || m_supported.contains(serviceName)) {
+      return true;
+    }
+
+    return false;
+  }
 }

ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java

@@ -317,6 +317,10 @@ public class UpgradeHelper {
       // !!! cluster and service checks are empty here
       for (UpgradePack.OrderService service : services) {
 
+        if (!context.isServiceSupported(service.serviceName)) {
+          continue;
+        }
+
         if (upgradePack.getType() == UpgradeType.ROLLING && !allTasks.containsKey(service.serviceName)) {
           continue;
         }

ambari-server/src/test/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheckTest.java

@@ -207,4 +207,61 @@ public class HostsRepositoryVersionCheckTest {
     Assert.assertEquals(PrereqCheckStatus.PASS, check.getStatus());
   }
 
+  @Test
+  public void testPerformWithVersionNotRequired() throws Exception {
+    final HostsRepositoryVersionCheck hostsRepositoryVersionCheck = new HostsRepositoryVersionCheck();
+    hostsRepositoryVersionCheck.clustersProvider = new Provider<Clusters>() {
+
+      @Override
+      public Clusters get() {
+        return clusters;
+      }
+    };
+    hostsRepositoryVersionCheck.repositoryVersionDaoProvider = new Provider<RepositoryVersionDAO>() {
+      @Override
+      public RepositoryVersionDAO get() {
+        return repositoryVersionDAO;
+      }
+    };
+    hostsRepositoryVersionCheck.hostVersionDaoProvider = new Provider<HostVersionDAO>() {
+      @Override
+      public HostVersionDAO get() {
+        return hostVersionDAO;
+      }
+    };
+
+    final Cluster cluster = Mockito.mock(Cluster.class);
+    Mockito.when(cluster.getClusterId()).thenReturn(1L);
+    Mockito.when(cluster.getDesiredStackVersion()).thenReturn(new StackId());
+    Mockito.when(clusters.getCluster("cluster")).thenReturn(cluster);
+    final Map<String, Host> hosts = new HashMap<String, Host>();
+    final Host host1 = Mockito.mock(Host.class);
+    final Host host2 = Mockito.mock(Host.class);
+    final Host host3 = Mockito.mock(Host.class);
+    Mockito.when(host1.getMaintenanceState(1L)).thenReturn(MaintenanceState.OFF);
+    Mockito.when(host2.getMaintenanceState(1L)).thenReturn(MaintenanceState.OFF);
+    Mockito.when(host3.getMaintenanceState(1L)).thenReturn(MaintenanceState.OFF);
+    hosts.put("host1", host1);
+    hosts.put("host2", host2);
+    hosts.put("host3", host3);
+    Mockito.when(clusters.getHostsForCluster("cluster")).thenReturn(hosts);
+
+    RepositoryVersionEntity rve = new RepositoryVersionEntity();
+    rve.setVersion("1.1.1");
+
+    HostVersionEntity hve = new HostVersionEntity();
+    hve.setRepositoryVersion(rve);
+    hve.setState(RepositoryVersionState.NOT_REQUIRED);
+
+    Mockito.when(
+        hostVersionDAO.findByHost(Mockito.anyString())).thenReturn(
+            Collections.singletonList(hve));
+
+    PrerequisiteCheck check = new PrerequisiteCheck(null, null);
+    PrereqCheckRequest request = new PrereqCheckRequest("cluster");
+    request.setRepositoryVersion("1.1.1");
+    hostsRepositoryVersionCheck.perform(check, request);
+    Assert.assertEquals(PrereqCheckStatus.PASS, check.getStatus());
+  }
+
 }

ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java

@@ -61,10 +61,12 @@ import org.apache.ambari.server.state.stack.upgrade.StageWrapper;
 import org.apache.ambari.server.state.stack.upgrade.Task;
 import org.apache.ambari.server.state.stack.upgrade.TaskWrapper;
 import org.apache.ambari.server.state.stack.upgrade.UpgradeType;
+import org.apache.ambari.server.utils.EventBusSynchronizer;
 import org.easymock.EasyMock;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.springframework.security.core.context.SecurityContextHolder;
 
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
@@ -74,7 +76,6 @@ import com.google.inject.Injector;
 import com.google.inject.Module;
 import com.google.inject.persist.PersistService;
 import com.google.inject.util.Modules;
-import org.springframework.security.core.context.SecurityContextHolder;
 
 /**
  * Tests the {@link UpgradeHelper} class
@@ -123,6 +124,8 @@ public class UpgradeHelperTest {
     // create an injector which will inject the mocks
     injector = Guice.createInjector(Modules.override(injectorModule).with(mockModule));
     injector.getInstance(GuiceJpaInitializer.class);
+    EventBusSynchronizer.synchronizeAmbariEventPublisher(injector);
+    EventBusSynchronizer.synchronizeAlertEventPublisher(injector);
 
     helper = injector.getInstance(OrmTestHelper.class);
     ambariMetaInfo = injector.getInstance(AmbariMetaInfo.class);
@@ -256,6 +259,58 @@ public class UpgradeHelperTest {
     assertEquals(8, groups.get(3).items.size());
   }
 
+  @Test
+  public void testSupportedServiceUpgradeOrchestration() throws Exception {
+    Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("foo", "bar");
+    assertTrue(upgrades.isEmpty());
+
+    upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1");
+
+    ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER");
+    si.setDisplayName("Zk");
+    ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER");
+    ci.setDisplayName("ZooKeeper1 Server2");
+
+    assertTrue(upgrades.containsKey("upgrade_test"));
+    UpgradePack upgrade = upgrades.get("upgrade_test");
+    assertNotNull(upgrade);
+
+    makeCluster();
+
+    UpgradeContext context = new UpgradeContext(m_masterHostResolver, HDP_21,
+        HDP_21, UPGRADE_VERSION, Direction.UPGRADE, UpgradeType.ROLLING);
+    context.setSupportedServices(Collections.singleton("ZOOKEEPER"));
+
+
+    List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
+
+    assertEquals(3, groups.size());
+
+    assertEquals("PRE_CLUSTER", groups.get(0).name);
+    assertEquals("ZOOKEEPER", groups.get(1).name);
+    assertEquals("POST_CLUSTER", groups.get(2).name);
+
+
+    UpgradeGroupHolder group = groups.get(1);
+    // check that the display name is being used
+    assertTrue(group.items.get(1).getText().contains("ZooKeeper1 Server2"));
+    assertEquals(group.items.get(5).getText(), "Service Check Zk");
+
+    UpgradeGroupHolder postGroup = groups.get(2);
+    assertEquals("POST_CLUSTER", postGroup.name);
+    assertEquals("Finalize Upgrade", postGroup.title);
+    assertEquals(3, postGroup.items.size());
+    assertEquals("Confirm Finalize", postGroup.items.get(0).getText());
+    assertEquals("Execute HDFS Finalize", postGroup.items.get(1).getText());
+    assertEquals("Save Cluster State", postGroup.items.get(2).getText());
+    assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType());
+
+    assertEquals(4, groups.get(0).items.size());
+    assertEquals(6, groups.get(1).items.size());
+    assertEquals(3, groups.get(2).items.size());
+  }
+
+
   @Test
   public void testUpgradeServerActionOrchestration() throws Exception {
     Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1");