AMBARI-12688 HDFS check fails after move NameNode on NN HA cluster (dsen)

ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java

@@ -43,6 +43,7 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -178,11 +179,15 @@ public class LogicalRequest extends Request {
     return new ArrayList<HostRequest>(allHostRequests);
   }
 
-  public Map<String, Collection<String>> getProjectedTopology() {
+  public Map<String, Collection<String>> getPendingHostComponents() {
     Map<String, Collection<String>> hostComponentMap = new HashMap<String, Collection<String>>();
 
+    Set<HostRequest> pendingRequests =
+      new HashSet<HostRequest>(requestsWithReservedHosts.values());
+    pendingRequests.addAll(outstandingHostRequests);
+
     //todo: synchronization
-    for (HostRequest hostRequest : allHostRequests) {
+    for (HostRequest hostRequest : pendingRequests) {
       HostGroup hostGroup = hostRequest.getHostGroup();
       for (String host : topology.getHostGroupInfo().get(hostGroup.getName()).getHostNames()) {
         Collection<String> hostComponents = hostComponentMap.get(host);

ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java

@@ -287,12 +287,13 @@ public class TopologyManager {
     return clusterTopologyMap.get(clusterName);
   }
 
-  public Map<String, Collection<String>> getProjectedTopology() {
+  public Map<String, Collection<String>> getPendingHostComponents() {
     ensureInitialized();
     Map<String, Collection<String>> hostComponentMap = new HashMap<String, Collection<String>>();
 
     for (LogicalRequest logicalRequest : allRequests.values()) {
-      Map<String, Collection<String>> requestTopology = logicalRequest.getProjectedTopology();
+      Map<String, Collection<String>> requestTopology =
+        logicalRequest.getPendingHostComponents();
       for (Map.Entry<String, Collection<String>> entry : requestTopology.entrySet()) {
         String host = entry.getKey();
         Collection<String> hostComponents = hostComponentMap.get(host);

ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java

@@ -277,7 +277,8 @@ public class StageUtils {
     }
 
     // add hosts from topology manager
-    Map<String, Collection<String>> pendingHostComponents = topologyManager.getProjectedTopology();
+    Map<String, Collection<String>> pendingHostComponents =
+      topologyManager.getPendingHostComponents();
     for (String hostname : pendingHostComponents.keySet()) {
       if (!hostsSet.contains(hostname)) {
         hostsSet.add(hostname);

ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java

@@ -194,7 +194,7 @@ public class KerberosHelperTest extends EasyMockSupport {
     setClusterController();
     //todo: StageUtils shouldn't be called for this test
     StageUtils.setTopologyManager(topologyManager);
-    expect(topologyManager.getProjectedTopology()).andReturn(
+    expect(topologyManager.getPendingHostComponents()).andReturn(
         Collections.<String, Collection<String>>emptyMap()).anyTimes();
   }
 

ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java

@@ -35,6 +35,7 @@ import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -307,7 +308,7 @@ public class StageUtilsTest extends EasyMockSupport {
     final ServiceComponentHost nns7ServiceComponentHost = createMock(ServiceComponentHost.class);
     expect(nns7ServiceComponentHost.getComponentAdminState()).andReturn(HostComponentAdminState.INSERVICE).anyTimes();
 
-    Map<String, Collection<String>> projectedTopology = new HashMap<String, Collection<String>>();
+    Map<String, Collection<String>> pendingHostComponents = new HashMap<String, Collection<String>>();
 
 
     final HashMap<String, ServiceComponentHost> nnServiceComponentHosts = new HashMap<String, ServiceComponentHost>() {
@@ -315,14 +316,14 @@ public class StageUtilsTest extends EasyMockSupport {
         put("h0", nnh0ServiceComponentHost);
       }
     };
-    insertTopology(projectedTopology, "NAMENODE", nnServiceComponentHosts.keySet());
+    insertTopology(pendingHostComponents, "NAMENODE", nnServiceComponentHosts.keySet());
 
     final HashMap<String, ServiceComponentHost> snnServiceComponentHosts = new HashMap<String, ServiceComponentHost>() {
       {
         put("h1", snnh1ServiceComponentHost);
       }
     };
-    insertTopology(projectedTopology, "SECONDARY_NAMENODE", snnServiceComponentHosts.keySet());
+    insertTopology(pendingHostComponents, "SECONDARY_NAMENODE", snnServiceComponentHosts.keySet());
 
     final Map<String, ServiceComponentHost> dnServiceComponentHosts = new HashMap<String, ServiceComponentHost>() {
       {
@@ -336,14 +337,14 @@ public class StageUtilsTest extends EasyMockSupport {
         put("h9", dnh9ServiceComponentHost);
       }
     };
-    insertTopology(projectedTopology, "DATANODE", dnServiceComponentHosts.keySet());
+    insertTopology(pendingHostComponents, "DATANODE", dnServiceComponentHosts.keySet());
 
     final Map<String, ServiceComponentHost> hbmServiceComponentHosts = new HashMap<String, ServiceComponentHost>() {
       {
         put("h5", hbm5ServiceComponentHost);
       }
     };
-    insertTopology(projectedTopology, "HBASE_MASTER", hbmServiceComponentHosts.keySet());
+    insertTopology(pendingHostComponents, "HBASE_MASTER", hbmServiceComponentHosts.keySet());
 
     final Map<String, ServiceComponentHost> hbrsServiceComponentHosts = new HashMap<String, ServiceComponentHost>() {
       {
@@ -354,14 +355,14 @@ public class StageUtilsTest extends EasyMockSupport {
         put("h9", hbrs9ServiceComponentHost);
       }
     };
-    insertTopology(projectedTopology, "HBASE_REGIONSERVER", hbrsServiceComponentHosts.keySet());
+    insertTopology(pendingHostComponents, "HBASE_REGIONSERVER", hbrsServiceComponentHosts.keySet());
 
     final Map<String, ServiceComponentHost> mrjtServiceComponentHosts = new HashMap<String, ServiceComponentHost>() {
       {
         put("h5", mrjt5ServiceComponentHost);
       }
     };
-    insertTopology(projectedTopology, "JOBTRACKER", mrjtServiceComponentHosts.keySet());
+    insertTopology(pendingHostComponents, "JOBTRACKER", mrjtServiceComponentHosts.keySet());
 
     final Map<String, ServiceComponentHost> mrttServiceComponentHosts = new HashMap<String, ServiceComponentHost>() {
       {
@@ -374,7 +375,7 @@ public class StageUtilsTest extends EasyMockSupport {
         put("h9", mrtt9ServiceComponentHost);
       }
     };
-    insertTopology(projectedTopology, "TASKTRACKER", mrttServiceComponentHosts.keySet());
+    insertTopology(pendingHostComponents, "TASKTRACKER", mrttServiceComponentHosts.keySet());
 
 
     final Map<String, ServiceComponentHost> nnsServiceComponentHosts = new HashMap<String, ServiceComponentHost>() {
@@ -382,7 +383,7 @@ public class StageUtilsTest extends EasyMockSupport {
         put("h7", nns7ServiceComponentHost);
       }
     };
-    insertTopology(projectedTopology, "NONAME_SERVER", nnsServiceComponentHosts.keySet());
+    insertTopology(pendingHostComponents, "NONAME_SERVER", nnsServiceComponentHosts.keySet());
 
     final ServiceComponent nnComponent = createMock(ServiceComponent.class);
     expect(nnComponent.getName()).andReturn("NAMENODE").anyTimes();
@@ -528,8 +529,8 @@ public class StageUtilsTest extends EasyMockSupport {
 
 
     final TopologyManager topologyManager = injector.getInstance(TopologyManager.class);
-    topologyManager.getProjectedTopology();
-    expectLastCall().andReturn(projectedTopology).once();
+    topologyManager.getPendingHostComponents();
+    expectLastCall().andReturn(Collections.emptyMap()).once();
 
     replayAll();
 
@@ -548,14 +549,14 @@ public class StageUtilsTest extends EasyMockSupport {
       assertTrue(allHosts.contains(host.getHostName()));
     }
 
-    checkServiceHostIndexes(info, "DATANODE", "slave_hosts", projectedTopology, hostNames);
-    checkServiceHostIndexes(info, "NAMENODE", "namenode_host", projectedTopology, hostNames);
-    checkServiceHostIndexes(info, "SECONDARY_NAMENODE", "snamenode_host", projectedTopology, hostNames);
-    checkServiceHostIndexes(info, "HBASE_MASTER", "hbase_master_hosts", projectedTopology, hostNames);
-    checkServiceHostIndexes(info, "HBASE_REGIONSERVER", "hbase_rs_hosts", projectedTopology, hostNames);
-    checkServiceHostIndexes(info, "JOBTRACKER", "jtnode_host", projectedTopology, hostNames);
-    checkServiceHostIndexes(info, "TASKTRACKER", "mapred_tt_hosts", projectedTopology, hostNames);
-    checkServiceHostIndexes(info, "NONAME_SERVER", "noname_server_hosts", projectedTopology, hostNames);
+    checkServiceHostIndexes(info, "DATANODE", "slave_hosts", pendingHostComponents, hostNames);
+    checkServiceHostIndexes(info, "NAMENODE", "namenode_host", pendingHostComponents, hostNames);
+    checkServiceHostIndexes(info, "SECONDARY_NAMENODE", "snamenode_host", pendingHostComponents, hostNames);
+    checkServiceHostIndexes(info, "HBASE_MASTER", "hbase_master_hosts", pendingHostComponents, hostNames);
+    checkServiceHostIndexes(info, "HBASE_REGIONSERVER", "hbase_rs_hosts", pendingHostComponents, hostNames);
+    checkServiceHostIndexes(info, "JOBTRACKER", "jtnode_host", pendingHostComponents, hostNames);
+    checkServiceHostIndexes(info, "TASKTRACKER", "mapred_tt_hosts", pendingHostComponents, hostNames);
+    checkServiceHostIndexes(info, "NONAME_SERVER", "noname_server_hosts", pendingHostComponents, hostNames);
 
     Set<String> actualPingPorts = info.get(StageUtils.PORTS);
     if (pingPorts.contains(null)) {
@@ -589,14 +590,14 @@ public class StageUtilsTest extends EasyMockSupport {
     // Validate substitutions...
     info = StageUtils.substituteHostIndexes(info);
 
-    checkServiceHostNames(info, "DATANODE", "slave_hosts", projectedTopology);
-    checkServiceHostNames(info, "NAMENODE", "namenode_host", projectedTopology);
-    checkServiceHostNames(info, "SECONDARY_NAMENODE", "snamenode_host", projectedTopology);
-    checkServiceHostNames(info, "HBASE_MASTER", "hbase_master_hosts", projectedTopology);
-    checkServiceHostNames(info, "HBASE_REGIONSERVER", "hbase_rs_hosts", projectedTopology);
-    checkServiceHostNames(info, "JOBTRACKER", "jtnode_host", projectedTopology);
-    checkServiceHostNames(info, "TASKTRACKER", "mapred_tt_hosts", projectedTopology);
-    checkServiceHostNames(info, "NONAME_SERVER", "noname_server_hosts", projectedTopology);
+    checkServiceHostNames(info, "DATANODE", "slave_hosts", pendingHostComponents);
+    checkServiceHostNames(info, "NAMENODE", "namenode_host", pendingHostComponents);
+    checkServiceHostNames(info, "SECONDARY_NAMENODE", "snamenode_host", pendingHostComponents);
+    checkServiceHostNames(info, "HBASE_MASTER", "hbase_master_hosts", pendingHostComponents);
+    checkServiceHostNames(info, "HBASE_REGIONSERVER", "hbase_rs_hosts", pendingHostComponents);
+    checkServiceHostNames(info, "JOBTRACKER", "jtnode_host", pendingHostComponents);
+    checkServiceHostNames(info, "TASKTRACKER", "mapred_tt_hosts", pendingHostComponents);
+    checkServiceHostNames(info, "NONAME_SERVER", "noname_server_hosts", pendingHostComponents);
   }
 
   private void insertTopology(Map<String, Collection<String>> projectedTopology, String componentName, Set<String> hostNames) {
@@ -614,8 +615,12 @@ public class StageUtilsTest extends EasyMockSupport {
     }
   }
 
-  private void checkServiceHostIndexes(Map<String, Set<String>> info, String componentName, String mappedComponentName,
-                                       Map<String, Collection<String>> serviceTopology, List<String> hostList) {
+  private void checkServiceHostIndexes(Map<String, Set<String>> info,
+                                       String componentName,
+                                       String mappedComponentName,
+                                       Map<String, Collection<String>> serviceTopology,
+                                       List<String> hostList) {
+
     Set<Integer> expectedHostsList = new HashSet<Integer>();
     Set<Integer> actualHostsList = new HashSet<Integer>();