Powering off RM node increases API latency by a factor of 6. (odiachenko)

ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaHostProvider.java

@@ -34,4 +34,26 @@ public interface GangliaHostProvider {
    * @throws SystemException if unable to get the Ganglia server host name
    */
   public String getGangliaCollectorHostName(String clusterName) throws SystemException;
+  
+  /**
+   * Get the status of Ganglia server host for the given cluster name.
+   *
+   * @param clusterName the cluster name
+   *
+   * @return true if heartbeat with Ganglia server host wasn't lost
+   *
+   * @throws SystemException if unable to get the status of Ganglia server host
+   */
+  public boolean isGangliaCollectorHostLive(String clusterName) throws SystemException;
+  
+  /**
+   * Get the status of Ganglia server component for the given cluster name.
+   *
+   * @param clusterName the cluster name
+   *
+   * @return true if Ganglia server component is started
+   *
+   * @throws SystemException if unable to get the status of Ganglia server component
+   */
+  public boolean isGangliaCollectorComponentLive(String clusterName) throws SystemException;
 }

ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java

@@ -428,6 +428,20 @@ public abstract class GangliaPropertyProvider extends AbstractPropertyProvider {
           requestAll ? Collections.<String>emptySet() : metrics.keySet(), temporalInfo);
       BufferedReader reader = null;
       try {
+        
+        //Check if host is live
+        if (!hostProvider.isGangliaCollectorHostLive(clusterName)) {
+          LOG.info("Ganglia host is not live");
+            return Collections.emptySet();
+        }
+        
+        //Check if Ganglia server component is live
+        if (!hostProvider.isGangliaCollectorComponentLive(clusterName)) {
+          LOG.info("Ganglia server component is not live");
+            return Collections.emptySet();
+        }
+        
+
         reader = new BufferedReader(new InputStreamReader(
             getStreamProvider().readFrom(spec)));
 
@@ -515,6 +529,7 @@ public abstract class GangliaPropertyProvider extends AbstractPropertyProvider {
       return Collections.emptySet();
     }
 
+
     /**
      * Populate the given resource with the given Ganglia metric.
      *

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java

@@ -18,8 +18,14 @@
 
 package org.apache.ambari.server.controller.internal;
 
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.Role;
 import org.apache.ambari.server.configuration.ComponentSSLConfiguration;
 import org.apache.ambari.server.controller.AmbariServer;
+import org.apache.ambari.server.controller.HostRequest;
+import org.apache.ambari.server.controller.HostResponse;
+import org.apache.ambari.server.controller.ServiceComponentHostRequest;
+import org.apache.ambari.server.controller.ServiceComponentHostResponse;
 import org.apache.ambari.server.controller.ganglia.GangliaComponentPropertyProvider;
 import org.apache.ambari.server.controller.ganglia.GangliaHostComponentPropertyProvider;
 import org.apache.ambari.server.controller.ganglia.GangliaHostPropertyProvider;
@@ -34,7 +40,10 @@ import org.apache.ambari.server.controller.AmbariManagementController;
 import com.google.inject.Inject;
 import org.apache.ambari.server.controller.utilities.StreamProvider;
 import org.apache.ambari.server.state.DesiredConfig;
+import org.apache.ambari.server.state.HostState;
 import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.State;
+import org.apache.commons.collections.CollectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -231,6 +240,63 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
     checkInit();
     return clusterGangliaCollectorMap.get(clusterName);
   }
+  
+  @Override
+  public boolean isGangliaCollectorHostLive(String clusterName) throws SystemException {
+    
+    HostResponse gangliaCollectorHost = null;
+    
+    try {
+      HostRequest hostRequest = new HostRequest(null, clusterName, Collections.<String, String>emptyMap());
+      Set<HostResponse> hosts = managementController.getHosts(Collections.singleton(hostRequest));
+      
+      final String gangliaCollectorHostName = getGangliaCollectorHostName(clusterName);
+      
+      gangliaCollectorHost = (HostResponse) CollectionUtils.find(hosts, new org.apache.commons.collections.Predicate() {
+        
+        @Override
+        public boolean evaluate(Object hostResponse) {
+          return ((HostResponse) hostResponse).getHostname().equals(gangliaCollectorHostName);
+        }
+      });      
+    } catch (AmbariException e) {
+      LOG.debug("Error checking of Ganglia server host live status: ", e);
+      return false;
+    }
+    
+    LOG.debug("Host state: " + gangliaCollectorHost.getHostState());
+    
+    return !gangliaCollectorHost.getHostState().equals(HostState.HEARTBEAT_LOST.name());
+  }
+  
+  @Override
+  public boolean isGangliaCollectorComponentLive(String clusterName) throws SystemException {
+
+
+    ServiceComponentHostResponse gangliaCollectorHostComponent = null;
+    
+    try {
+      final String gangliaCollectorHostName = getGangliaCollectorHostName(clusterName);
+      ServiceComponentHostRequest componentRequest = new ServiceComponentHostRequest(clusterName, "GANGLIA",
+                                                                                     Role.GANGLIA_SERVER.name(),
+                                                                                     gangliaCollectorHostName,
+                                                                                     Collections.<String, String>emptyMap(), null);
+      
+      Set<ServiceComponentHostResponse> hostComponents = managementController.getHostComponents(Collections.singleton(componentRequest));
+       gangliaCollectorHostComponent = (ServiceComponentHostResponse) CollectionUtils.find(hostComponents,
+           new org.apache.commons.collections.Predicate() {
+        @Override
+        public boolean evaluate(Object arg0) {
+          return ((ServiceComponentHostResponse) arg0).getHostname().equals(gangliaCollectorHostName);
+        }
+      });
+    } catch (AmbariException e) {
+      LOG.debug("Error checking of Ganglia server host component state: ", e);
+      return false;
+    }
+    
+    return gangliaCollectorHostComponent.getLiveState().equals(State.STARTED.name());
+  }
 
 
   // ----- utility methods ---------------------------------------------------

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/URLStreamProvider.java

@@ -35,12 +35,7 @@ import javax.net.ssl.TrustManagerFactory;
 import org.apache.ambari.server.controller.utilities.StreamProvider;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.message.BasicHeader;
 
 /**
  * URL based implementation of a stream provider.

ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java

@@ -31,6 +31,11 @@ import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.anyObject;
+import org.powermock.api.easymock.PowerMock;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -45,6 +50,7 @@ import java.util.Set;
  * Test the Ganglia property provider.
  */
 @RunWith(Parameterized.class)
+@PrepareForTest({ GangliaHostProvider.class })
 public class GangliaPropertyProviderTest {
 
   private static final String PROPERTY_ID = PropertyHelper.getPropertyId("metrics/jvm", "gcCount");
@@ -161,6 +167,47 @@ public class GangliaPropertyProviderTest {
     Assert.assertNotNull(resource.getPropertyValue(shuffle_output_bytes));
     Assert.assertNotNull(resource.getPropertyValue(shuffle_success_outputs));
   }
+  
+  @Test
+  public void testPopulateResources_checkHostComponent() throws Exception {
+    TestStreamProvider streamProvider  = new TestStreamProvider("temporal_ganglia_data.txt");
+    GangliaHostProvider hostProvider =  PowerMock.createPartialMock(GangliaHostProvider.class,
+        "isGangliaCollectorHostLive", "isGangliaCollectorComponentLive");
+
+    GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
+        PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, PropertyHelper.MetricsVersion.HDP1),
+        streamProvider,
+        configuration,
+        hostProvider,
+        CLUSTER_NAME_PROPERTY_ID,
+        HOST_NAME_PROPERTY_ID,
+        COMPONENT_NAME_PROPERTY_ID);
+
+    // datanode
+    Resource resource = new ResourceImpl(Resource.Type.HostComponent);
+
+    resource.setProperty(HOST_NAME_PROPERTY_ID, "domU-12-31-39-0E-34-E1.compute-1.internal");
+    resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "DATANODE");
+
+    // only ask for one property
+    Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>();
+    temporalInfoMap.put(PROPERTY_ID, new TemporalInfoImpl(10L, 20L, 1L));
+    Request  request = PropertyHelper.getReadRequest(Collections.singleton(PROPERTY_ID), temporalInfoMap);
+    
+    expect(hostProvider.getGangliaCollectorHostName(anyObject(String.class))).andReturn("ganglia-host");
+    expect(hostProvider.isGangliaCollectorComponentLive(anyObject(String.class))).andReturn(true).once();
+    expect(hostProvider.isGangliaCollectorHostLive(anyObject(String.class))).andReturn(true).once();
+    
+    
+    PowerMock.replay(hostProvider);
+    
+    Set<Resource> populateResources = propertyProvider.populateResources(Collections.singleton(resource), request, null);
+    
+    PowerMock.verify(hostProvider);
+    
+    Assert.assertEquals(1, populateResources.size());
+    
+  }
 
 
   @Test
@@ -719,9 +766,33 @@ public class GangliaPropertyProviderTest {
 
   private static class TestGangliaHostProvider implements GangliaHostProvider {
 
+    private boolean isHostLive;
+    private boolean isComponentLive;
+    
+    public TestGangliaHostProvider() {
+      this(true, true);
+    }
+
+    public TestGangliaHostProvider(boolean isHostLive, boolean isComponentLive) {
+      this.isHostLive = isHostLive;
+      this.isComponentLive = isComponentLive;
+    }
+
     @Override
     public String getGangliaCollectorHostName(String clusterName) {
       return "domU-12-31-39-0E-34-E1.compute-1.internal";
     }
+
+    @Override
+    public boolean isGangliaCollectorHostLive(String clusterName)
+        throws SystemException {
+      return isHostLive;
+    }
+
+    @Override
+    public boolean isGangliaCollectorComponentLive(String clusterName)
+        throws SystemException {
+      return isComponentLive;
+    }
   }
 }

ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaReportPropertyProviderTest.java

@@ -23,6 +23,7 @@ import org.apache.ambari.server.controller.internal.ResourceImpl;
 import org.apache.ambari.server.controller.internal.TemporalInfoImpl;
 import org.apache.ambari.server.controller.spi.Request;
 import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.controller.spi.SystemException;
 import org.apache.ambari.server.controller.spi.TemporalInfo;
 import org.apache.ambari.server.controller.utilities.PropertyHelper;
 import org.junit.Assert;
@@ -106,5 +107,17 @@ public class GangliaReportPropertyProviderTest {
     public String getGangliaCollectorHostName(String clusterName) {
       return "domU-12-31-39-0E-34-E1.compute-1.internal";
     }
+
+    @Override
+    public boolean isGangliaCollectorHostLive(String clusterName)
+        throws SystemException {
+      return true;
+    }
+
+    @Override
+    public boolean isGangliaCollectorComponentLive(String clusterName)
+        throws SystemException {
+      return true;
+    }
   }
 }