AMBARI-2022 - Service Component metric collection API takes over a minute on large cluster

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/trunk@1471764 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -269,6 +269,9 @@ Trunk (unreleased changes):
 
  IMPROVEMENTS
 
+ AMBARI-2022. Service Component metric collection API takes over a minute
+ on large cluster. (tbeerbower)
+
  AMBARI-2005. When adding a component to a host (after cluster deployment),
  UI should warn that nagios server need to be restarted. (yusaku)
 

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java

@@ -42,6 +42,9 @@ import java.util.*;
  */
 public abstract class AbstractProviderModule implements ProviderModule, ResourceProviderObserver, JMXHostProvider, GangliaHostProvider {
 
+  private static final int PROPERTY_REQUEST_CONNECT_TIMEOUT = 5000;
+  private static final int PROPERTY_REQUEST_READ_TIMEOUT    = 10000;
+
   private static final String CLUSTER_NAME_PROPERTY_ID                  = PropertyHelper.getPropertyId("Clusters", "cluster_name");
   private static final String HOST_COMPONENT_CLUSTER_NAME_PROPERTY_ID   = PropertyHelper.getPropertyId("HostRoles", "cluster_name");
   private static final String HOST_COMPONENT_HOST_NAME_PROPERTY_ID      = PropertyHelper.getPropertyId("HostRoles", "host_name");
@@ -106,7 +109,7 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
   /**
    * JMX ports read from the configs
    */
-  private Map<String, Map<String, String>> jmxPortMap = Collections
+  private final Map<String, Map<String, String>> jmxPortMap = Collections
     .synchronizedMap(new HashMap<String, Map<String, String>>());
 
   private volatile boolean initialized = false;
@@ -175,6 +178,7 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
     Map<String,String> clusterJmxPorts = jmxPortMap.get(clusterName);
     if (clusterJmxPorts == null) {
       synchronized (jmxPortMap) {
+        clusterJmxPorts = jmxPortMap.get(clusterName);
         if (clusterJmxPorts == null) {
           clusterJmxPorts = new HashMap<String, String>();
           jmxPortMap.put(clusterName, clusterJmxPorts);
@@ -242,7 +246,8 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
 
     List<PropertyProvider> providers = new LinkedList<PropertyProvider>();
 
-    URLStreamProvider streamProvider = new URLStreamProvider();
+    URLStreamProvider streamProvider = new URLStreamProvider(
+        PROPERTY_REQUEST_CONNECT_TIMEOUT, PROPERTY_REQUEST_READ_TIMEOUT);
 
     switch (type){
       case Cluster :
@@ -268,7 +273,9 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
             this,
             PropertyHelper.getPropertyId("ServiceComponentInfo", "cluster_name"),
             null,
-            PropertyHelper.getPropertyId("ServiceComponentInfo", "component_name")));
+            PropertyHelper.getPropertyId("ServiceComponentInfo", "component_name"),
+            PropertyHelper.getPropertyId("ServiceComponentInfo", "state"),
+            Collections.singleton("STARTED")));
 
         providers.add(new GangliaComponentPropertyProvider(
             PropertyHelper.getGangliaPropertyIds(type),
@@ -284,7 +291,9 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
             this,
             PropertyHelper.getPropertyId("HostRoles", "cluster_name"),
             PropertyHelper.getPropertyId("HostRoles", "host_name"),
-            PropertyHelper.getPropertyId("HostRoles", "component_name")));
+            PropertyHelper.getPropertyId("HostRoles", "component_name"),
+            PropertyHelper.getPropertyId("HostRoles", "state"),
+            Collections.singleton("STARTED")));
 
         providers.add(new GangliaHostComponentPropertyProvider(
             PropertyHelper.getGangliaPropertyIds(type),
@@ -325,7 +334,7 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
     Request          request  = PropertyHelper.getReadRequest(CLUSTER_NAME_PROPERTY_ID);
 
     try {
-      jmxPortMap = new HashMap<String, Map<String, String>>();
+      jmxPortMap.clear();
       Set<Resource> clusters = provider.getResources(request, null);
 
       clusterHostComponentMap    = new HashMap<String, Map<String, String>>();
@@ -411,8 +420,8 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
     String versionTag = "version1";
     if (serviceResource != null) {
       for (Resource res : serviceResource) {
-        Map<String, String> configs = (Map<String,
-          String>) res.getPropertyValue(ServiceResourceProvider.SERVICE_DESIRED_CONFIGS_PROPERTY_ID);
+        Map<String, String> configs = (Map<String, String>)
+            res.getPropertyValue(ServiceResourceProvider.SERVICE_DESIRED_CONFIGS_PROPERTY_ID);
         if (configs != null) {
           versionTag = configs.get(configType);
         }
@@ -431,7 +440,7 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
       (ConfigurationResourceProvider.CONFIGURATION_CLUSTER_NAME_PROPERTY_ID).equals(clusterName).and()
       .property(ConfigurationResourceProvider.CONFIGURATION_CONFIG_TYPE_PROPERTY_ID).equals(configType).and()
       .property(ConfigurationResourceProvider.CONFIGURATION_CONFIG_TAG_PROPERTY_ID).equals(versionTag).toPredicate();
-    Set<Resource> configResources = null;
+    Set<Resource> configResources;
     try {
       configResources = configResourceProvider.getResources
         (PropertyHelper.getReadRequest(ConfigurationResourceProvider.CONFIGURATION_CLUSTER_NAME_PROPERTY_ID,
@@ -439,7 +448,7 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
           ConfigurationResourceProvider.CONFIGURATION_CONFIG_TAG_PROPERTY_ID), configPredicate);
     } catch (NoSuchResourceException e) {
       LOG.info("Resource for the desired config not found. " + e);
-      return Collections.EMPTY_MAP;
+      return Collections.emptyMap();
     }
     Map<String, String> mConfigs = new HashMap<String, String>();
     if (configResources != null) {

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java

@@ -68,17 +68,20 @@ class HostComponentResourceProvider extends AbstractControllerResourceProvider {
   
   //Component name mappings
   private static final Map<String, PropertyProvider> HOST_COMPONENT_PROPERTIES_PROVIDER = new HashMap<String, PropertyProvider>();
-  
-  private static final int HOST_COMPONENT_HTTP_PROPERTY_REQUEST_TIMEOUT = 1500;
-  
+
+  private static final int HOST_COMPONENT_HTTP_PROPERTY_REQUEST_CONNECT_TIMEOUT = 1500;
+  private static final int HOST_COMPONENT_HTTP_PROPERTY_REQUEST_READ_TIMEOUT    = 10000;
+
   static {
     HOST_COMPONENT_PROPERTIES_PROVIDER.put(
         "NAGIOS_SERVER",
-        new HttpProxyPropertyProvider(new URLStreamProvider(
-            HOST_COMPONENT_HTTP_PROPERTY_REQUEST_TIMEOUT), PropertyHelper
-            .getPropertyId("HostRoles", "cluster_name"), PropertyHelper
-            .getPropertyId("HostRoles", "host_name"), PropertyHelper
-            .getPropertyId("HostRoles", "component_name")));
+        new HttpProxyPropertyProvider(
+            new URLStreamProvider(
+              HOST_COMPONENT_HTTP_PROPERTY_REQUEST_CONNECT_TIMEOUT,
+              HOST_COMPONENT_HTTP_PROPERTY_REQUEST_READ_TIMEOUT),
+            PropertyHelper.getPropertyId("HostRoles", "cluster_name"),
+            PropertyHelper.getPropertyId("HostRoles", "host_name"),
+            PropertyHelper.getPropertyId("HostRoles", "component_name")));
   }
 
   //Parameters from the predicate

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/URLStreamProvider.java

@@ -29,28 +29,29 @@ import org.apache.ambari.server.controller.utilities.StreamProvider;
  * URL based implementation of a stream provider.
  */
 public class URLStreamProvider implements StreamProvider {
-  
-  private int connTimeout = -1;
-  
-  public URLStreamProvider() {
-  }
-  
+
+  private final int connTimeout;
+  private final int readTimeout;
+
   /**
    * Provide the connection timeout for the underlying connection.
-   * 
+   *
    * @param connectionTimeout time, in milliseconds, to attempt a connection
+   * @param readTimeout
    */
-  public URLStreamProvider(int connectionTimeout) {
-    connTimeout = connectionTimeout;
+  public URLStreamProvider(int connectionTimeout, int readTimeout) {
+    this.connTimeout = connectionTimeout;
+    this.readTimeout = readTimeout;
   }
   
   @Override
   public InputStream readFrom(String spec) throws IOException {
     URLConnection connection = new URL(spec).openConnection();
-    if (connTimeout > 0) {
-      connection.setConnectTimeout(connTimeout);
-    }
+
+    connection.setConnectTimeout(connTimeout);
+    connection.setReadTimeout(readTimeout);
     connection.setDoOutput(true);
+
     return connection.getInputStream();
   }
 }

ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXPropertyProvider.java

@@ -57,6 +57,10 @@ public class JMXPropertyProvider extends AbstractPropertyProvider {
 
   private final String componentNamePropertyId;
 
+  private final String statePropertyId;
+
+  private final Set<String> healthyStates;
+
   private final static ObjectReader objectReader;
 
 
@@ -88,13 +92,17 @@ public class JMXPropertyProvider extends AbstractPropertyProvider {
    * @param clusterNamePropertyId    the cluster name property id
    * @param hostNamePropertyId       the host name property id
    * @param componentNamePropertyId  the component name property id
+   * @param statePropertyId          the state property id
+   * @param healthyStates            the set of healthy state values
    */
   public JMXPropertyProvider(Map<String, Map<String, PropertyInfo>> componentMetrics,
                              StreamProvider streamProvider,
                              JMXHostProvider jmxHostProvider,
                              String clusterNamePropertyId,
                              String hostNamePropertyId,
-                             String componentNamePropertyId) {
+                             String componentNamePropertyId,
+                             String statePropertyId,
+                             Set<String> healthyStates) {
 
     super(componentMetrics);
 
@@ -103,6 +111,8 @@ public class JMXPropertyProvider extends AbstractPropertyProvider {
     this.clusterNamePropertyId    = clusterNamePropertyId;
     this.hostNamePropertyId       = hostNamePropertyId;
     this.componentNamePropertyId  = componentNamePropertyId;
+    this.statePropertyId          = statePropertyId;
+    this.healthyStates            = healthyStates;
   }
 
 
@@ -144,16 +154,25 @@ public class JMXPropertyProvider extends AbstractPropertyProvider {
    * @param request   the request
    * @param predicate the predicate
    *
-   * @return true if the resource was successfully populated with the requested properties
+   * @return true if the resource should be part of the result set for the given predicate
    */
   private boolean populateResource(Resource resource, Request request, Predicate predicate)
       throws SystemException {
 
     Set<String> ids = getRequestPropertyIds(request, predicate);
     if (ids.isEmpty()) {
+      // no properties requested
       return true;
     }
 
+    // Don't attempt to get the JMX properties if the resource is in an unhealthy state
+    if (statePropertyId != null) {
+      String state = (String) resource.getPropertyValue(statePropertyId);
+      if (state != null && !healthyStates.contains(state)) {
+        return true;
+      }
+    }
+
     String componentName = (String) resource.getPropertyValue(componentNamePropertyId);
 
     if (getComponentMetrics().get(componentName) == null) {

ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java

@@ -37,22 +37,29 @@ import java.util.Set;
 public class JMXPropertyProviderTest {
   protected static final String HOST_COMPONENT_HOST_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "host_name");
   protected static final String HOST_COMPONENT_COMPONENT_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "component_name");
+  protected static final String HOST_COMPONENT_STATE_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "state");
 
   @Test
-  public void testGetResources() throws Exception {
+  public void testPopulateResources() throws Exception {
     TestStreamProvider  streamProvider = new TestStreamProvider();
     TestJMXHostProvider hostProvider = new TestJMXHostProvider(false);
 
     JMXPropertyProvider propertyProvider = new JMXPropertyProvider(
         PropertyHelper.getJMXPropertyIds(Resource.Type.HostComponent),
         streamProvider,
-        hostProvider, PropertyHelper.getPropertyId("HostRoles", "cluster_name"), PropertyHelper.getPropertyId("HostRoles", "host_name"), PropertyHelper.getPropertyId("HostRoles", "component_name"));
+        hostProvider,
+        PropertyHelper.getPropertyId("HostRoles", "cluster_name"),
+        PropertyHelper.getPropertyId("HostRoles", "host_name"),
+        PropertyHelper.getPropertyId("HostRoles", "component_name"),
+        PropertyHelper.getPropertyId("HostRoles", "state"),
+        Collections.singleton("STARTED"));
 
     // namenode
     Resource resource = new ResourceImpl(Resource.Type.HostComponent);
 
     resource.setProperty(HOST_COMPONENT_HOST_NAME_PROPERTY_ID, "domu-12-31-39-0e-34-e1.compute-1.internal");
     resource.setProperty(HOST_COMPONENT_COMPONENT_NAME_PROPERTY_ID, "NAMENODE");
+    resource.setProperty(HOST_COMPONENT_STATE_PROPERTY_ID, "STARTED");
 
     // request with an empty set should get all supported properties
     Request request = PropertyHelper.getReadRequest(Collections.<String>emptySet());
@@ -194,6 +201,7 @@ public class JMXPropertyProviderTest {
 
     resource.setProperty(HOST_COMPONENT_HOST_NAME_PROPERTY_ID, "domu-12-31-39-14-ee-b3.compute-1.internal");
     resource.setProperty(HOST_COMPONENT_COMPONENT_NAME_PROPERTY_ID, "HBASE_MASTER");
+    resource.setProperty(HOST_COMPONENT_STATE_PROPERTY_ID, "STARTED");
 
     // only ask for specific properties
     properties = new HashSet<String>();
@@ -208,7 +216,7 @@ public class JMXPropertyProviderTest {
 
     Assert.assertEquals(propertyProvider.getSpec("domu-12-31-39-14-ee-b3.compute-1.internal", "60010"), streamProvider.getLastSpec());
 
-    Assert.assertEquals(7, PropertyHelper.getProperties(resource).size());
+    Assert.assertEquals(8, PropertyHelper.getProperties(resource).size());
     Assert.assertEquals(1069416448, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryMax")));
     Assert.assertEquals(4806976, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryUsed")));
     Assert.assertEquals(136314880, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryMax")));
@@ -219,14 +227,19 @@ public class JMXPropertyProviderTest {
   }
 
   @Test
-  public void testGetResourcesWithUnknownPort() throws Exception {
+  public void testPopulateResourcesWithUnknownPort() throws Exception {
     TestStreamProvider  streamProvider = new TestStreamProvider();
     TestJMXHostProvider hostProvider = new TestJMXHostProvider(true);
 
     JMXPropertyProvider propertyProvider = new JMXPropertyProvider(
         PropertyHelper.getJMXPropertyIds(Resource.Type.HostComponent),
         streamProvider,
-        hostProvider, PropertyHelper.getPropertyId("HostRoles", "cluster_name"), PropertyHelper.getPropertyId("HostRoles", "host_name"), PropertyHelper.getPropertyId("HostRoles", "component_name"));
+        hostProvider,
+        PropertyHelper.getPropertyId("HostRoles", "cluster_name"),
+        PropertyHelper.getPropertyId("HostRoles", "host_name"),
+        PropertyHelper.getPropertyId("HostRoles", "component_name"),
+        PropertyHelper.getPropertyId("HostRoles", "state"),
+        Collections.singleton("STARTED"));
 
     // namenode
     Resource resource = new ResourceImpl(Resource.Type.HostComponent);
@@ -250,6 +263,37 @@ public class JMXPropertyProviderTest {
     Assert.assertEquals(23634400, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryUsed")));
   }
 
+  @Test
+  public void testPopulateResourcesUnhealthyResource() throws Exception {
+    TestStreamProvider  streamProvider = new TestStreamProvider();
+    TestJMXHostProvider hostProvider = new TestJMXHostProvider(true);
+
+    JMXPropertyProvider propertyProvider = new JMXPropertyProvider(
+        PropertyHelper.getJMXPropertyIds(Resource.Type.HostComponent),
+        streamProvider,
+        hostProvider,
+        PropertyHelper.getPropertyId("HostRoles", "cluster_name"),
+        PropertyHelper.getPropertyId("HostRoles", "host_name"),
+        PropertyHelper.getPropertyId("HostRoles", "component_name"),
+        PropertyHelper.getPropertyId("HostRoles", "state"),
+        Collections.singleton("STARTED"));
+
+    // namenode
+    Resource resource = new ResourceImpl(Resource.Type.HostComponent);
+
+    resource.setProperty(HOST_COMPONENT_HOST_NAME_PROPERTY_ID, "domu-12-31-39-0e-34-e1.compute-1.internal");
+    resource.setProperty(HOST_COMPONENT_COMPONENT_NAME_PROPERTY_ID, "NAMENODE");
+    resource.setProperty(HOST_COMPONENT_STATE_PROPERTY_ID, "INSTALLED");
+
+    // request with an empty set should get all supported properties
+    Request request = PropertyHelper.getReadRequest(Collections.<String>emptySet());
+
+    Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
+
+    // Assert that the stream provider was never called.
+    Assert.assertNull(streamProvider.getLastSpec());
+  }
+
   private static class TestJMXHostProvider implements JMXHostProvider {
     private final boolean unknownPort;