AMBARI-6263 API call to /hosts to get information about 10 hosts takes more than 30 seconds on 2K-node cluster (dsen)

ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java

@@ -76,6 +76,11 @@ public class QueryImpl implements Query, ResourceInstance {
    */
   Map<Resource, QueryResult> queryResults = new LinkedHashMap<Resource, QueryResult>();
 
+  /**
+   * Set of populated query results
+   */
+  Map<Resource, QueryResult> populatedQueryResults = new LinkedHashMap<Resource, QueryResult>();
+
   /**
    * Sub-resources of the resource which is being operated on.
    * Should only be added via {@link #addSubResource(String, QueryImpl)}
@@ -337,13 +342,28 @@ public class QueryImpl implements Query, ResourceInstance {
     Request       request     = createRequest();
     Set<Resource> resourceSet = new LinkedHashSet<Resource>();
 
-    for (Resource queryResource : doQuery(resourceType, request, queryPredicate)) {
-      providerResourceSet.add(queryResource);
-      resourceSet.add(queryResource);
+    Set<Resource> queryResources = doQuery(resourceType, request, queryPredicate);
+    // If there is a page request and there is no user predicate
+    if ((pageRequest != null || sortRequest != null )&&
+      userPredicate == null) {
+      PageResponse pageResponse = clusterController.getPage(resourceType,
+        queryResources, request, queryPredicate, pageRequest, sortRequest);
+
+      // build a new set
+      for (Resource r : pageResponse.getIterable()) {
+        resourceSet.add(r);
+        providerResourceSet.add(r);
+      }
+    } else {
+      resourceSet.addAll(queryResources);
+      providerResourceSet.addAll(queryResources);
     }
 
+    populatedQueryResults.put(null, new QueryResult(
+      request, queryPredicate, userPredicate, getKeyValueMap(), resourceSet));
+
     queryResults.put(null, new QueryResult(
-        request, queryPredicate, userPredicate, getKeyValueMap(), resourceSet));
+      request, queryPredicate, userPredicate, getKeyValueMap(), queryResources));
 
     clusterController.populateResources(resourceType, providerResourceSet, request, queryPredicate);
     queryForSubResources();
@@ -365,7 +385,7 @@ public class QueryImpl implements Query, ResourceInstance {
       Request       request             = subResource.createRequest();
       Set<Resource> providerResourceSet = new HashSet<Resource>();
 
-      for (QueryResult queryResult : queryResults.values()) {
+      for (QueryResult queryResult : populatedQueryResults.values()) {
         for (Resource resource : queryResult.getProviderResourceSet()) {
           Map<Resource.Type, String> map = getKeyValueMap(resource, queryResult.getKeyValueMap());
 
@@ -373,15 +393,16 @@ public class QueryImpl implements Query, ResourceInstance {
           Set<Resource> resourceSet    = new LinkedHashSet<Resource>();
 
           try {
-            for (Resource queryResource : subResource.doQuery(resourceType, request, queryPredicate)) {
-              providerResourceSet.add(queryResource);
-              resourceSet.add(queryResource);
-            }
+            Set<Resource> queryResources = subResource.doQuery(resourceType, request, queryPredicate);
+            providerResourceSet.addAll(queryResources);
+            resourceSet.addAll(queryResources);
           } catch (NoSuchResourceException e) {
             // do nothing ...
           }
           subResource.queryResults.put(resource,
               new QueryResult(request, queryPredicate, subResourcePredicate, map, resourceSet));
+          subResource.populatedQueryResults.put(resource,
+            new QueryResult(request, queryPredicate, subResourcePredicate, map, resourceSet));
         }
       }
       clusterController.populateResources(resourceType, providerResourceSet, request, null);
@@ -475,7 +496,7 @@ public class QueryImpl implements Query, ResourceInstance {
     Map<String, String> categoryPropertyIdMap =
         getPropertyIdsForCategory(propertyIds, category);
 
-    for (Map.Entry<Resource, QueryResult> queryResultEntry : queryResults.entrySet()) {
+    for (Map.Entry<Resource, QueryResult> queryResultEntry : populatedQueryResults.entrySet()) {
       QueryResult queryResult         = queryResultEntry.getValue();
       Resource    queryParentResource = queryResultEntry.getKey();
 

ambari-server/src/main/java/org/apache/ambari/server/api/services/BaseRequest.java

@@ -244,14 +244,14 @@ public abstract class BaseRequest implements Request {
       if(from.equals("start")) {
         startingPoint = PageRequest.StartingPoint.Beginning;
       } else {
-        offset = Integer.valueOf(from);
+        offset = Integer.parseInt(from);
         startingPoint = PageRequest.StartingPoint.OffsetStart;
       }
     } else if (to != null ) {
       if (to.equals("end")) {
         startingPoint = PageRequest.StartingPoint.End;
       } else {
-        offset = Integer.valueOf(to);
+        offset = Integer.parseInt(to);
         startingPoint = PageRequest.StartingPoint.OffsetEnd;
       }
     } else {

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java

@@ -70,8 +70,8 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
   private static final String GANGLIA_SERVER                            = "GANGLIA_SERVER";
   private static final String PROPERTIES_CATEGORY = "properties";
   private static final Map<Service.Type, String> serviceConfigVersions = new ConcurrentHashMap<Service.Type, String>();
-  private static final Map<Service.Type, String> serviceConfigTypes = new HashMap<Service.Type, String>();
-  private static final Map<Service.Type, Map<String, String[]>> serviceDesiredProperties = new HashMap<Service.Type, Map<String, String[]>>();
+  private static final Map<Service.Type, String> serviceConfigTypes = new EnumMap<Service.Type, String>(Service.Type.class);
+  private static final Map<Service.Type, Map<String, String[]>> serviceDesiredProperties = new EnumMap<Service.Type, Map<String, String[]>>(Service.Type.class);
   private static final Map<String, Service.Type> componentServiceMap = new HashMap<String, Service.Type>();
   
   private static final Map<String, Map<String, String[]>> jmxDesiredProperties = new HashMap<String, Map<String,String[]>>();
@@ -244,12 +244,12 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
             currVersion, serviceConfigTypes.get(service),
             serviceDesiredProperties.get(service));
           
-          for (String compName : portMap.keySet()) {
+          for (Entry<String, String> entry : portMap.entrySet()) {
             // portString will be null if the property defined for the component doesn't exist
             // this will trigger using the default port for the component
-            String portString = getPortString(portMap.get(compName));
+            String portString = getPortString(entry.getValue());
             if (null != portString)
-              clusterJmxPorts.put(compName, portString);
+              clusterJmxPorts.put(entry.getKey(), portString);
           }
         }
       } catch (Exception e) {
@@ -279,17 +279,17 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
           Map<String, String> refMap = new HashMap<String, String>();
           while(refValueString.contains("${")) {
               int startValueRef = refValueString.indexOf("${") + 2;
-              int endValueRef = refValueString.indexOf("}");
+              int endValueRef = refValueString.indexOf('}');
               String valueRef = refValueString.substring(startValueRef, endValueRef);
               refValueString = refValueString.substring(endValueRef+1);
-              String trueValue = (String) postProcessPropertyValue(valueRef, properties.get(valueRef), properties, prevProps);
+              String trueValue = postProcessPropertyValue(valueRef, properties.get(valueRef), properties, prevProps);
               if (trueValue != null){
-               refMap.put("${"+valueRef+"}", trueValue);
+               refMap.put("${"+valueRef+ '}', trueValue);
               } 
           }
-          for (String keyRef : refMap.keySet()){
-            refValueString = refMap.get(keyRef);
-            value = ((String)value).replace(keyRef, refValueString);
+          for (Entry<String, String> entry : refMap.entrySet()){
+            refValueString = entry.getValue();
+            value = value.replace(entry.getKey(), refValueString);
           }
           properties.put(key, value);
     }
@@ -655,7 +655,7 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
     Map<String, String> mConfigs = new HashMap<String, String>();
     if (configResources != null) {
       for (Resource res : configResources) {
-       Map<String, String> evalutedProperties = null;                      
+       Map<String, String> evaluatedProperties = null;
         for (Entry<String,String[]> entry : keys.entrySet()) {
           String propName = null;
           String value = null;
@@ -669,8 +669,8 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
           }
           
           if (value != null && value.contains("${")) {
-            if (evalutedProperties == null){
-              evalutedProperties = new HashMap<String, String>();
+            if (evaluatedProperties == null){
+              evaluatedProperties = new HashMap<String, String>();
               Map<String, Object> properties = res.getPropertiesMap().get(PROPERTIES_CATEGORY);
               for (Map.Entry<String, Object> subentry : properties.entrySet()) {
                 String keyString = subentry.getKey();
@@ -678,13 +678,13 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
                 String valueString;
                 if (object != null && object instanceof String){
                   valueString = (String)object;
-                  evalutedProperties.put(keyString, valueString);
-                  postProcessPropertyValue(keyString, valueString, evalutedProperties, null);
+                  evaluatedProperties.put(keyString, valueString);
+                  postProcessPropertyValue(keyString, valueString, evaluatedProperties, null);
                 }
               }              
             }
           }
-          value = postProcessPropertyValue(propName, value, evalutedProperties, null);
+          value = postProcessPropertyValue(propName, value, evaluatedProperties, null);
           LOG.debug("PROPERTY -> key: " + propName + ", " + "value: " + value);
           
           mConfigs.put(entry.getKey(), value);

ambari-server/src/main/java/org/apache/ambari/server/controller/spi/ClusterController.java

@@ -33,9 +33,9 @@ public interface ClusterController extends SchemaFactory {
    * Get the resources of the given type filtered by the given request and
    * predicate objects.
    *
-   * @param type      the type of the requested resources
-   * @param request   the request object which defines the desired set of properties
-   * @param predicate the predicate object which filters which resources are returned
+   * @param type        the type of the requested resources
+   * @param request     the request object which defines the desired set of properties
+   * @param predicate   the predicate object which filters which resources are returned
    *
    * @return an iterable object of the requested resources
    *
@@ -78,6 +78,8 @@ public interface ClusterController extends SchemaFactory {
    * @param providerResources  set of populated Resources
    * @param request            the request
    * @param predicate          the predicate object which filters which resources are returned
+   * @param pageRequest        the page request for a paginated response
+   * @param sortRequest        the sortRequest object which defines if the resources need to be sorted
    *
    * @return a page response representing the requested page of resources
    *

ambari-server/src/main/java/org/apache/ambari/server/utils/JaxbMapKeyListAdapter.java

@@ -34,8 +34,8 @@ public class JaxbMapKeyListAdapter extends
     }
     JaxbMapKeyList[] list = new JaxbMapKeyList[map.size()] ;
     int index = 0;
-    for (String key : map.keySet()) {
-      JaxbMapKeyList jaxbMap = new JaxbMapKeyList(key, map.get(key));
+    for (Map.Entry<String, List<String>> stringListEntry : map.entrySet()) {
+      JaxbMapKeyList jaxbMap = new JaxbMapKeyList(stringListEntry.getKey(), stringListEntry.getValue());
       list[index++] = jaxbMap;
     }
     return list;

ambari-server/src/main/java/org/apache/ambari/server/utils/JaxbMapKeyMapAdapter.java

@@ -35,10 +35,10 @@ public class JaxbMapKeyMapAdapter extends
     }
     JaxbMapKeyMap[] list = new JaxbMapKeyMap[map.size()];
     int index=0;
-    for (String key : map.keySet()) {
-      Map<String, String> value = map.get(key);
+    for (Map.Entry<String, Map<String, String>> stringMapEntry : map.entrySet()) {
+      Map<String, String> value = stringMapEntry.getValue();
       JaxbMapKeyVal[] keyValList = mapAdapter.marshal(value);
-      list[index++] = new JaxbMapKeyMap(key, keyValList);
+      list[index++] = new JaxbMapKeyMap(stringMapEntry.getKey(), keyValList);
     }
     return list;
   }

ambari-server/src/main/java/org/apache/ambari/server/utils/JaxbMapKeyValAdapter.java

@@ -32,8 +32,8 @@ public class JaxbMapKeyValAdapter extends
     }
     JaxbMapKeyVal[] list = new JaxbMapKeyVal[m.size()] ;
     int index = 0;
-    for (String key : m.keySet()) {
-      JaxbMapKeyVal jaxbMap = new JaxbMapKeyVal(key, m.get(key));
+    for (Map.Entry<String, String> entry : m.entrySet()) {
+      JaxbMapKeyVal jaxbMap = new JaxbMapKeyVal(entry.getKey(), entry.getValue());
       list[index++] = jaxbMap;
     }
     return list;