AMBARI-5020. Lost heartbeat on host but ganglia shows a heartbeat lost (dlysnichenko)

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java

@@ -853,7 +853,10 @@ public class ServiceResourceProvider extends AbstractControllerResourceProvider
             Set<ServiceComponentHostResponse> hostComponentResponses =
                 controller.getHostComponents(Collections.singleton(request));
 
-            State   serviceState = null;
+            State   masterState = null;
+            State   clientState = null;
+            State   otherState = null;
+
             boolean hasDisabled  = false;
             boolean hasMaster    = false;
             boolean hasOther     = false;
@@ -864,7 +867,7 @@ public class ServiceResourceProvider extends AbstractControllerResourceProvider
                   stackId.getStackVersion(), hostComponentResponse.getServiceName(),
                   hostComponentResponse.getComponentName());
 
-            if (componentInfo != null) {
+              if (componentInfo != null) {
                 State state = getHostComponentState(hostComponentResponse);
 
                 if (state.equals(State.DISABLED)) {
@@ -873,29 +876,28 @@ public class ServiceResourceProvider extends AbstractControllerResourceProvider
                   if (componentInfo.isMaster()) {
                     hasMaster = true;
                     if(!state.equals(State.STARTED) &&
-                        ( serviceState == null || state.ordinal() > serviceState.ordinal())) {
-                      serviceState = state;
+                        ( masterState == null || state.ordinal() > masterState.ordinal())) {
+                      masterState = state;
                     }
                   } else if (componentInfo.isClient()) {
                     hasClient = true;
-                    if (!(hasMaster || hasOther) && !state.equals(State.INSTALLED) &&
-                        (serviceState == null || state.ordinal() > serviceState.ordinal())) {
-                      serviceState = state;
+                    if (!state.equals(State.INSTALLED) &&
+                        (clientState == null || state.ordinal() > clientState.ordinal())) {
+                      clientState = state;
                     }
                   } else {
                     hasOther  = true;
-                    if (!hasMaster &&
-                        (serviceState == null || state.ordinal() > serviceState.ordinal())) {
-                      serviceState = state;
+                    if (otherState == null || state.ordinal() > otherState.ordinal()) {
+                      otherState = state;
                     }
                   }
                 }
               }
             }
 
-            return hasMaster   ? serviceState == null ? State.STARTED : serviceState :
-                   hasOther    ? serviceState :
-                   hasClient   ? serviceState == null ? State.INSTALLED : serviceState :
+            return hasMaster   ? masterState == null ? State.STARTED : masterState :
+                   hasOther    ? otherState :
+                   hasClient   ? clientState == null ? State.INSTALLED : clientState :
                    hasDisabled ? State.DISABLED : State.UNKNOWN;
           }
         } catch (AmbariException e) {

ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ServiceResourceProviderTest.java

@@ -949,6 +949,54 @@ public class ServiceResourceProviderTest {
     verify(managementController, clusters, cluster, ambariMetaInfo, stackId, componentInfo);
   }
 
+  @Test
+  public void testGangliaServiceState_ArbitraryComponentOrder_STARTED() throws Exception{
+    AmbariManagementController managementController = createMock(AmbariManagementController.class);
+    Clusters clusters = createNiceMock(Clusters.class);
+    Cluster cluster = createNiceMock(Cluster.class);
+    AmbariMetaInfo ambariMetaInfo = createNiceMock(AmbariMetaInfo.class);
+    StackId stackId = createNiceMock(StackId.class);
+    ComponentInfo componentInfo = createNiceMock(ComponentInfo.class);
+
+    ServiceComponentHostResponse shr1 = new ServiceComponentHostResponse("C1", "GANGLIA", "GANGLIA_MONITOR", "Host100", "STARTED", "", null, null, null);
+    ServiceComponentHostResponse shr2 = new ServiceComponentHostResponse("C1", "GANGLIA", "GANGLIA_MONITOR", "Host199", "UNKNOWN", "", null, null, null);
+    ServiceComponentHostResponse shr3 = new ServiceComponentHostResponse("C1", "GANGLIA", "GANGLIA_SERVER", "Host100", "STARTED", "", null, null, null);
+
+    Set<ServiceComponentHostResponse> responses = new LinkedHashSet<ServiceComponentHostResponse>();
+    responses.add(shr1);
+    responses.add(shr2);
+    responses.add(shr3);
+
+    // set expectations
+    expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes();
+    expect(managementController.getClusters()).andReturn(clusters).anyTimes();
+    expect(clusters.getCluster("C1")).andReturn(cluster).anyTimes();
+    expect(componentInfo.isMaster()).andReturn(false).once();
+    expect(componentInfo.isMaster()).andReturn(false).once();
+    expect(componentInfo.isMaster()).andReturn(true).once();
+    expect(componentInfo.isClient()).andReturn(false).anyTimes();
+    expect(managementController.getHostComponents((Set<ServiceComponentHostRequest>) anyObject())).andReturn(responses).anyTimes();
+    expect(cluster.getDesiredStackVersion()).andReturn(stackId);
+
+    expect(stackId.getStackName()).andReturn("S1").anyTimes();
+    expect(stackId.getStackVersion()).andReturn("V1").anyTimes();
+
+
+    expect(ambariMetaInfo.getComponentCategory((String) anyObject(), (String) anyObject(),
+            (String) anyObject(), (String) anyObject())).andReturn(componentInfo).anyTimes();
+
+    // replay
+    replay(managementController, clusters, cluster, ambariMetaInfo, stackId, componentInfo);
+
+    ServiceResourceProvider.ServiceState serviceState = new ServiceResourceProvider.DefaultServiceState();
+
+    State state = serviceState.getState(managementController, "C1", "GANGLIA");
+    Assert.assertEquals(State.STARTED, state);
+
+    // verify
+    verify(managementController, clusters, cluster, ambariMetaInfo, stackId, componentInfo);
+  }
+
   @Test
   public void testDefaultServiceState_ClientOnly_INSTALLED() throws Exception{
     AmbariManagementController managementController = createMock(AmbariManagementController.class);