AMBARI-6489. Add sequence for components start, in case of resourcemanager HA mode enabled.(vbrodetskyi)

ambari-server/src/main/java/org/apache/ambari/server/metadata/RoleCommandOrder.java

@@ -50,7 +50,8 @@ public class RoleCommandOrder {
   private final static String GENERAL_DEPS_KEY = "general_deps";
   private final static String GLUSTERFS_DEPS_KEY = "optional_glusterfs";
   private final static String NO_GLUSTERFS_DEPS_KEY = "optional_no_glusterfs";
-  private final static String HA_DEPS_KEY = "optional_ha";
+  private final static String NAMENODE_HA_DEPS_KEY = "namenode_optional_ha";
+  private final static String RESOURCEMANAGER_HA_DEPS_KEY = "resourcemanager_optional_ha";
   private final static String COMMENT_STR = "_comment";
   private static final String ROLE_COMMAND_ORDER_FILE = "role_command_order.json";
 
@@ -172,7 +173,8 @@ public class RoleCommandOrder {
 
   public void initialize(Cluster cluster) {
     Boolean hasGLUSTERFS = false;
-    Boolean isHAEnabled = false;
+    Boolean isNameNodeHAEnabled = false;
+    Boolean isResourceManagerHAEnabled = false;
 
     try {
       if (cluster != null && cluster.getService("GLUSTERFS") != null) {
@@ -185,7 +187,16 @@ public class RoleCommandOrder {
       if (cluster != null &&
               cluster.getService("HDFS") != null &&
               cluster.getService("HDFS").getServiceComponent("JOURNALNODE") != null) {
-        isHAEnabled = true;
+        isNameNodeHAEnabled = true;
+      }
+    } catch (AmbariException e) {
+    }
+
+    try {
+      if (cluster != null &&
+              cluster.getService("YARN") != null &&
+              cluster.getService("YARN").getServiceComponent("RESOURCEMANAGER").getServiceComponentHosts().size() > 1) {
+        isResourceManagerHAEnabled = true;
       }
     } catch (AmbariException e) {
     }
@@ -226,9 +237,13 @@ public class RoleCommandOrder {
       Map<String,Object> noGlusterFSSection = (Map<String, Object>) userData.get(NO_GLUSTERFS_DEPS_KEY);
       addDependencies(noGlusterFSSection);
     }
-    if (isHAEnabled) {
-      Map<String,Object> isHASection = (Map<String, Object>) userData.get(HA_DEPS_KEY);
-      addDependencies(isHASection);
+    if (isNameNodeHAEnabled) {
+      Map<String,Object> NAMENODEHASection = (Map<String, Object>) userData.get(NAMENODE_HA_DEPS_KEY);
+      addDependencies(NAMENODEHASection);
+    }
+    if (isResourceManagerHAEnabled) {
+      Map<String,Object> ResourceManagerHASection = (Map<String, Object>) userData.get(RESOURCEMANAGER_HA_DEPS_KEY);
+      addDependencies(ResourceManagerHASection);
     }
     extendTransitiveDependency();
   }

ambari-server/src/main/resources/role_command_order.json

@@ -90,11 +90,15 @@
     "JOBTRACKER-UPGRADE": ["HDFS_CLIENT-UPGRADE"]
   },
   "_comment" : "Dependencies that are used in HA NameNode cluster",
-  "optional_ha": {
+  "namenode_optional_ha": {
     "NAMENODE-START": ["JOURNALNODE-START", "ZOOKEEPER_SERVER-START"],
     "ZKFC-START": ["NAMENODE-START"],
     "NAGIOS_SERVER-START": ["ZKFC-START", "JOURNALNODE-START"],
     "HDFS_SERVICE_CHECK-SERVICE_CHECK": ["ZKFC-START"]
+  },
+  "_comment" : "Dependencies that are used in ResourceManager HA cluster",
+  "resourcemanager_optional_ha" : {
+    "RESOURCEMANAGER-START": ["ZOOKEEPER_SERVER-START"]
   }
 }
 

ambari-server/src/main/resources/stacks/HDP/1.3.2/role_command_order.json

@@ -89,11 +89,15 @@
     "JOBTRACKER-UPGRADE": ["HDFS_CLIENT-UPGRADE"]
   },
   "_comment" : "Dependencies that are used in HA NameNode cluster",
-  "optional_ha": {
+  "namenode_optional_ha": {
     "NAMENODE-START": ["JOURNALNODE-START", "ZOOKEEPER_SERVER-START"],
     "ZKFC-START": ["NAMENODE-START"],
     "NAGIOS_SERVER-START": ["ZKFC-START", "JOURNALNODE-START"],
     "HDFS_SERVICE_CHECK-SERVICE_CHECK": ["ZKFC-START"]
+  },
+  "_comment" : "Dependencies that are used in ResourceManager HA cluster",
+  "resourcemanager_optional_ha" : {
+    "RESOURCEMANAGER-START": ["ZOOKEEPER_SERVER-START"]
   }
 }
 

ambari-server/src/main/resources/stacks/HDP/1.3.3/role_command_order.json

@@ -89,11 +89,15 @@
     "JOBTRACKER-UPGRADE": ["HDFS_CLIENT-UPGRADE"]
   },
   "_comment" : "Dependencies that are used in HA NameNode cluster",
-  "optional_ha": {
+  "namenode_optional_ha": {
     "NAMENODE-START": ["JOURNALNODE-START", "ZOOKEEPER_SERVER-START"],
     "ZKFC-START": ["NAMENODE-START"],
     "NAGIOS_SERVER-START": ["ZKFC-START", "JOURNALNODE-START"],
     "HDFS_SERVICE_CHECK-SERVICE_CHECK": ["ZKFC-START"]
+  },
+  "_comment" : "Dependencies that are used in ResourceManager HA cluster",
+  "resourcemanager_optional_ha" : {
+    "RESOURCEMANAGER-START": ["ZOOKEEPER_SERVER-START"]
   }
 }
 

ambari-server/src/main/resources/stacks/HDP/1.3/role_command_order.json

@@ -89,11 +89,15 @@
     "JOBTRACKER-UPGRADE": ["HDFS_CLIENT-UPGRADE"]
   },
   "_comment" : "Dependencies that are used in HA NameNode cluster",
-  "optional_ha": {
+  "namenode_optional_ha": {
     "NAMENODE-START": ["JOURNALNODE-START", "ZOOKEEPER_SERVER-START"],
     "ZKFC-START": ["NAMENODE-START"],
     "NAGIOS_SERVER-START": ["ZKFC-START", "JOURNALNODE-START"],
     "HDFS_SERVICE_CHECK-SERVICE_CHECK": ["ZKFC-START"]
+  },
+  "_comment" : "Dependencies that are used in ResourceManager HA cluster",
+  "resourcemanager_optional_ha" : {
+    "RESOURCEMANAGER-START": ["ZOOKEEPER_SERVER-START"]
   }
 }
 

ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/role_command_order.json

@@ -90,11 +90,15 @@
     "JOBTRACKER-UPGRADE": ["HDFS_CLIENT-UPGRADE"]
   },
   "_comment" : "Dependencies that are used in HA NameNode cluster",
-  "optional_ha": {
+  "namenode_optional_ha": {
     "NAMENODE-START": ["JOURNALNODE-START", "ZOOKEEPER_SERVER-START"],
     "ZKFC-START": ["NAMENODE-START"],
     "NAGIOS_SERVER-START": ["ZKFC-START", "JOURNALNODE-START"],
     "HDFS_SERVICE_CHECK-SERVICE_CHECK": ["ZKFC-START"]
+  },
+  "_comment" : "Dependencies that are used in ResourceManager HA cluster",
+  "resourcemanager_optional_ha" : {
+    "RESOURCEMANAGER-START": ["ZOOKEEPER_SERVER-START"]
   }
 }
 

ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json

@@ -90,10 +90,14 @@
     "JOBTRACKER-UPGRADE": ["HDFS_CLIENT-UPGRADE"]
   },
   "_comment" : "Dependencies that are used in HA NameNode cluster",
-  "optional_ha": {
+  "namenode_optional_ha": {
     "NAMENODE-START": ["ZKFC-START", "JOURNALNODE-START", "ZOOKEEPER_SERVER-START"],
     "ZKFC-START": ["ZOOKEEPER_SERVER-START"],
     "NAGIOS_SERVER-START": ["ZKFC-START", "JOURNALNODE-START"]
+  },
+  "_comment" : "Dependencies that are used in ResourceManager HA cluster",
+  "resourcemanager_optional_ha" : {
+    "RESOURCEMANAGER-START": ["ZOOKEEPER_SERVER-START"]
   }
 }
 

ambari-server/src/main/resources/stacks/HDP/2.0/role_command_order.json

@@ -90,10 +90,14 @@
     "JOBTRACKER-UPGRADE": ["HDFS_CLIENT-UPGRADE"]
   },
   "_comment" : "Dependencies that are used in HA NameNode cluster",
-  "optional_ha": {
+  "namenode_optional_ha": {
     "NAMENODE-START": ["ZKFC-START", "JOURNALNODE-START", "ZOOKEEPER_SERVER-START"],
     "ZKFC-START": ["ZOOKEEPER_SERVER-START"],
     "NAGIOS_SERVER-START": ["ZKFC-START", "JOURNALNODE-START"]
+  },
+  "_comment" : "Dependencies that are used in ResourceManager HA cluster",
+  "resourcemanager_optional_ha" : {
+    "RESOURCEMANAGER-START": ["ZOOKEEPER_SERVER-START"]
   }
 }
 

ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/role_command_order.json

@@ -101,10 +101,14 @@
     "JOBTRACKER-UPGRADE": ["HDFS_CLIENT-UPGRADE"]
   },
   "_comment" : "Dependencies that are used in HA NameNode cluster",
-  "optional_ha": {
+  "namenode_optional_ha": {
     "NAMENODE-START": ["ZKFC-START", "JOURNALNODE-START", "ZOOKEEPER_SERVER-START"],
     "ZKFC-START": ["ZOOKEEPER_SERVER-START"],
     "NAGIOS_SERVER-START": ["ZKFC-START", "JOURNALNODE-START"]
+  },
+  "_comment" : "Dependencies that are used in ResourceManager HA cluster",
+  "resourcemanager_optional_ha" : {
+    "RESOURCEMANAGER-START": ["ZOOKEEPER_SERVER-START"]
   }
 }
 

ambari-server/src/main/resources/stacks/HDP/2.1/role_command_order.json

@@ -101,10 +101,14 @@
     "JOBTRACKER-UPGRADE": ["HDFS_CLIENT-UPGRADE"]
   },
   "_comment" : "Dependencies that are used in HA NameNode cluster",
-  "optional_ha": {
+  "namenode_optional_ha": {
     "NAMENODE-START": ["ZKFC-START", "JOURNALNODE-START", "ZOOKEEPER_SERVER-START"],
     "ZKFC-START": ["ZOOKEEPER_SERVER-START"],
     "NAGIOS_SERVER-START": ["ZKFC-START", "JOURNALNODE-START"]
+  },
+  "_comment" : "Dependencies that are used in ResourceManager HA cluster",
+  "resourcemanager_optional_ha" : {
+    "RESOURCEMANAGER-START": ["ZOOKEEPER_SERVER-START"]
   }
 }
 

ambari-server/src/test/java/org/apache/ambari/server/metadata/RoleCommandOrderTest.java

@@ -21,6 +21,7 @@ package org.apache.ambari.server.metadata;
 import static junit.framework.Assert.assertEquals;
 import static junit.framework.Assert.assertFalse;
 import static junit.framework.Assert.assertTrue;
+import static junit.framework.Assert.assertNotNull;
 import static org.easymock.EasyMock.createMock;
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.replay;
@@ -28,18 +29,23 @@ import static org.easymock.EasyMock.verify;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.Role;
+import org.apache.ambari.server.RoleCommand;
 import org.apache.ambari.server.metadata.RoleCommandOrder.RoleCommandPair;
 import org.apache.ambari.server.orm.GuiceJpaInitializer;
 import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
 import org.apache.ambari.server.state.Service;
 import org.apache.ambari.server.state.ServiceComponent;
+import org.apache.ambari.server.state.ServiceComponentHost;
 import org.apache.ambari.server.state.StackId;
 import org.apache.ambari.server.state.cluster.ClusterImpl;
+import org.apache.ambari.server.state.svccomphost.ServiceComponentHostImpl;
 import org.codehaus.jackson.annotate.JsonAutoDetect;
 import org.codehaus.jackson.annotate.JsonMethod;
 import org.codehaus.jackson.map.ObjectMapper;
@@ -83,6 +89,7 @@ public class RoleCommandOrderTest {
     expect(cluster.getCurrentStackVersion()).andReturn(new StackId("HDP", "2.0.6"));
     expect(cluster.getService("GLUSTERFS")).andReturn(service);
     expect(cluster.getService("HDFS")).andReturn(null);
+    expect(cluster.getService("YARN")).andReturn(null);
     replay(cluster);
 
     Map<RoleCommandPair, Set<RoleCommandPair>> deps = rco.getDependencies();
@@ -128,6 +135,7 @@ public class RoleCommandOrderTest {
     Service hdfsService = createMock(Service.class);
 
     expect(cluster.getService("HDFS")).andReturn(hdfsService).atLeastOnce();
+    expect(cluster.getService("YARN")).andReturn(null);
     expect(hdfsService.getServiceComponent("JOURNALNODE")).andReturn(null);
     expect(cluster.getCurrentStackVersion()).andReturn(new StackId("HDP", "2.0.6"));
 
@@ -172,6 +180,7 @@ public class RoleCommandOrderTest {
     ServiceComponent journalnodeSC = createMock(ServiceComponent.class);
 
     expect(cluster.getService("HDFS")).andReturn(hdfsService).atLeastOnce();
+    expect(cluster.getService("YARN")).andReturn(null);
     expect(hdfsService.getServiceComponent("JOURNALNODE")).andReturn(journalnodeSC);
     expect(cluster.getCurrentStackVersion()).andReturn(new StackId("HDP", "2.0.6"));
 
@@ -201,6 +210,60 @@ public class RoleCommandOrderTest {
     assertTrue(dependenciesContainBlockedRole(deps, Role.ZKFC));
   }
 
+  @Test
+  public void testInitializeAtHaRMCluster() throws AmbariException {
+    RoleCommandOrder rco = injector.getInstance(RoleCommandOrder.class);
+    ClusterImpl cluster = createMock(ClusterImpl.class);
+    ServiceComponentHost sch1 = createMock(ServiceComponentHostImpl.class);
+    ServiceComponentHost sch2 = createMock(ServiceComponentHostImpl.class);
+    expect(cluster.getService("GLUSTERFS")).andReturn(null);
+
+
+    Map<String, ServiceComponentHost> hostComponents = new HashMap<String, ServiceComponentHost>();
+    hostComponents.put("1",sch1);
+    hostComponents.put("2",sch2);
+
+    Service yarnService = createMock(Service.class);
+    ServiceComponent resourcemanagerSC = createMock(ServiceComponent.class);
+
+    expect(cluster.getService("YARN")).andReturn(yarnService).atLeastOnce();
+    expect(cluster.getService("HDFS")).andReturn(null);
+    expect(yarnService.getServiceComponent("RESOURCEMANAGER")).andReturn(resourcemanagerSC).anyTimes();
+    expect(resourcemanagerSC.getServiceComponentHosts()).andReturn(hostComponents).anyTimes();
+    expect(cluster.getCurrentStackVersion()).andReturn(new StackId("HDP", "2.0.6"));
+
+    replay(cluster, yarnService, sch1, sch2, resourcemanagerSC);
+
+    Map<RoleCommandPair, Set<RoleCommandPair>> deps = rco.getDependencies();
+    assertTrue("Dependencies are empty before initialization", deps.size() == 0);
+    rco.initialize(cluster);
+    assertTrue("Dependencies are loaded after initialization", deps.size() > 0);
+    verify(cluster, yarnService);
+    // Check that GLUSTERFS components are not present in dependencies
+    // Checking blocked roles
+    assertFalse(dependenciesContainBlockedRole(deps, Role.PEERSTATUS));
+    assertFalse(dependenciesContainBlockedRole(deps, Role.GLUSTERFS_SERVICE_CHECK));
+    assertFalse(dependenciesContainBlockedRole(deps, Role.GLUSTERFS_CLIENT));
+    // Checking blocker roles
+    assertFalse(dependenciesContainBlockerRole(deps, Role.PEERSTATUS));
+    assertFalse(dependenciesContainBlockerRole(deps, Role.GLUSTERFS_SERVICE_CHECK));
+    assertFalse(dependenciesContainBlockerRole(deps, Role.GLUSTERFS_CLIENT));
+
+    // And that some HDFS components are present (section has been loaded)
+    assertTrue(dependenciesContainBlockerRole(deps, Role.DATANODE));
+    // Check that some HA RM dependencies are present
+    RoleCommandPair rmPair = new RoleCommandPair(Role.RESOURCEMANAGER, RoleCommand.START);
+    Set<RoleCommandPair> rmRoleCommandPairs = deps.get(rmPair);
+    assertNotNull(rmRoleCommandPairs);
+    boolean isZookeeperStartPresent = false;
+    for (RoleCommandPair pair : rmRoleCommandPairs) {
+      if (pair.cmd == RoleCommand.START && pair.getRole() == Role.ZOOKEEPER_SERVER) {
+        isZookeeperStartPresent = true;
+      }
+    }
+    assertTrue(isZookeeperStartPresent);
+  }
+
 
   @Test
   public void testAddDependencies() throws IOException {
@@ -240,6 +303,7 @@ public class RoleCommandOrderTest {
     Service hdfsService = createMock(Service.class);
 
     expect(cluster.getService("HDFS")).andReturn(hdfsService).atLeastOnce();
+    expect(cluster.getService("YARN")).andReturn(null);
     expect(hdfsService.getServiceComponent("JOURNALNODE")).andReturn(null);
     //There is no rco file in this stack, should use default
     expect(cluster.getCurrentStackVersion()).andReturn(new StackId("HDP", "2.0.5"));

ambari-server/src/test/resources/stacks/HDP/2.0.6/role_command_order.json

@@ -90,11 +90,15 @@
     "JOBTRACKER-UPGRADE": ["HDFS_CLIENT-UPGRADE"]
   },
   "_comment" : "Dependencies that are used in HA NameNode cluster",
-  "optional_ha": {
+  "namenode_optional_ha": {
     "NAMENODE-START": ["JOURNALNODE-START", "ZOOKEEPER_SERVER-START"],
     "ZKFC-START": ["NAMENODE-START"],
     "NAGIOS_SERVER-START": ["ZKFC-START", "JOURNALNODE-START"],
     "HDFS_SERVICE_CHECK-SERVICE_CHECK": ["ZKFC-START"]
+  },
+  "_comment" : "Dependencies that are used in ResourceManager HA cluster",
+  "resourcemanager_optional_ha" : {
+    "RESOURCEMANAGER-START": ["ZOOKEEPER_SERVER-START"]
   }
 }
 

ambari-server/src/test/resources/stacks/HDP/2.0.7/role_command_order.json

@@ -90,11 +90,15 @@
     "JOBTRACKER-UPGRADE": ["HDFS_CLIENT-UPGRADE"]
   },
   "_comment" : "Dependencies that are used in HA NameNode cluster",
-  "optional_ha": {
+  "namenode_optional_ha": {
     "NAMENODE-START": ["JOURNALNODE-START", "ZOOKEEPER_SERVER-START"],
     "ZKFC-START": ["NAMENODE-START"],
     "NAGIOS_SERVER-START": ["ZKFC-START", "JOURNALNODE-START"],
     "HDFS_SERVICE_CHECK-SERVICE_CHECK": ["ZKFC-START"]
+  },
+  "_comment" : "Dependencies that are used in ResourceManager HA cluster",
+  "resourcemanager_optional_ha" : {
+    "RESOURCEMANAGER-START": ["ZOOKEEPER_SERVER-START"]
   }
 }
 

ambari-server/src/test/resources/stacks/HDP/2.0.8/role_command_order.json

@@ -90,11 +90,15 @@
     "JOBTRACKER-UPGRADE": ["HDFS_CLIENT-UPGRADE"]
   },
   "_comment" : "Dependencies that are used in HA NameNode cluster",
-  "optional_ha": {
+  "namenode_optional_ha": {
     "NAMENODE-START": ["JOURNALNODE-START", "ZOOKEEPER_SERVER-START"],
     "ZKFC-START": ["NAMENODE-START"],
     "NAGIOS_SERVER-START": ["ZKFC-START", "JOURNALNODE-START"],
     "HDFS_SERVICE_CHECK-SERVICE_CHECK": ["ZKFC-START"]
+  },
+  "_comment" : "Dependencies that are used in ResourceManager HA cluster",
+  "resourcemanager_optional_ha" : {
+    "RESOURCEMANAGER-START": ["ZOOKEEPER_SERVER-START"]
   }
 }
 

ambari-server/src/test/resources/stacks/HDP/2.1.1/role_command_order.json

@@ -90,11 +90,15 @@
     "JOBTRACKER-UPGRADE": ["HDFS_CLIENT-UPGRADE"]
   },
   "_comment" : "Dependencies that are used in HA NameNode cluster",
-  "optional_ha": {
+  "namenode_optional_ha": {
     "NAMENODE-START": ["JOURNALNODE-START", "ZOOKEEPER_SERVER-START"],
     "ZKFC-START": ["NAMENODE-START"],
     "NAGIOS_SERVER-START": ["ZKFC-START", "JOURNALNODE-START"],
     "HDFS_SERVICE_CHECK-SERVICE_CHECK": ["ZKFC-START"]
+  },
+  "_comment" : "Dependencies that are used in ResourceManager HA cluster",
+  "resourcemanager_optional_ha" : {
+    "RESOURCEMANAGER-START": ["ZOOKEEPER_SERVER-START"]
   }
 }