AMBARI-3017. Service metrics call latency increases by a factor of 2 or 3 every 20-30 calls. (mpapirkovskyy)

ambari-server/src/main/java/org/apache/ambari/server/api/query/Query.java

@@ -92,4 +92,5 @@ public interface Query {
    * @param pageRequest  the page request information
    */
   public void setPageRequest(PageRequest pageRequest);
+
 }

ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java

@@ -20,6 +20,7 @@ package org.apache.ambari.server.api.query;
 
 import org.apache.ambari.server.api.resources.ResourceInstance;
 import org.apache.ambari.server.api.services.ResultImpl;
+import org.apache.ambari.server.api.util.TreeNodeImpl;
 import org.apache.ambari.server.controller.utilities.ClusterControllerHelper;
 import org.apache.ambari.server.controller.utilities.PropertyHelper;
 import org.apache.ambari.server.controller.predicate.AndPredicate;
@@ -77,6 +78,21 @@ public class QueryImpl implements Query {
    */
   private PageRequest m_pageRequest;
 
+  /**
+   * Predicate calculated for query
+   */
+  private Predicate m_predicate;
+
+  /**
+   * Query resources, don't clone
+   */
+  private Set<Resource> m_providerResources;
+
+  /**
+   * Populated query resources with applied predicate and pagination, don't clone
+   */
+  private Iterable<Resource> m_resourceIterable;
+
   /**
    * The logger.
    */
@@ -121,64 +137,179 @@ public class QueryImpl implements Query {
     m_setQueryProperties.add(property);
   }
 
-  @Override
-  public Result execute()
-      throws UnsupportedPropertyException, SystemException, NoSuchResourceException, NoSuchParentResourceException {
+  protected Set<Resource> getProviderResources() throws NoSuchParentResourceException, UnsupportedPropertyException, NoSuchResourceException, SystemException {
+    if (m_providerResources == null) {
+      Resource.Type resourceType = m_resource.getResourceDefinition().getType();
 
-    Result result = createResult();
-    Resource.Type resourceType = m_resource.getResourceDefinition().getType();
-    if (m_resource.getIds().get(resourceType) == null) {
-      addCollectionProperties(resourceType);
-      result.getResultTree().setProperty("isCollection", "true");
+      if (m_resource.getIds().get(getResourceType()) == null) {
+        addCollectionProperties(getResourceType());
+      }
+
+      if (m_setQueryProperties.isEmpty() && m_mapSubResources.isEmpty()) {
+        //Add sub resource properties for default case where no fields are specified.
+        m_mapSubResources.putAll(m_resource.getSubResources());
+      }
+
+      if (LOG.isDebugEnabled()) {
+        //todo: include predicate info.  Need to implement toString for all predicates.
+        LOG.debug("Executing resource query: " + m_resource.getIds());
+      }
+
+      Predicate predicate = getPredicate();
+
+      Request request = createRequest();
+      m_providerResources = getClusterController().getRawResources(resourceType, request, predicate);
     }
 
-    if (m_setQueryProperties.isEmpty() && m_mapSubResources.isEmpty()) {
-      //Add sub resource properties for default case where no fields are specified.
-      m_mapSubResources.putAll(m_resource.getSubResources());
+    return m_providerResources;
+  }
+
+  protected Iterable<Resource> getResourceIterable() throws NoSuchParentResourceException, UnsupportedPropertyException,
+      NoSuchResourceException, SystemException {
+    if (m_resourceIterable == null) {
+      if (m_pageRequest == null) {
+        m_resourceIterable = getClusterController()
+            .getResources(getResourceType(), m_providerResources, getPredicate());
+      } else {
+        m_resourceIterable = getClusterController()
+            .getResources(getResourceType(), m_providerResources, getPredicate(), m_pageRequest).getIterable();
+      }
     }
+    return m_resourceIterable;
+  }
+
+  @Override
+  public Result execute() throws NoSuchParentResourceException, UnsupportedPropertyException, NoSuchResourceException, SystemException {
+
+    Map<Resource,Collection<Resource>> resourceTree =
+        new HashMap<Resource, Collection<Resource>>();
+    Map<Resource,String> resourceNames = new HashMap<Resource, String>();
+
+    Map<QueryImpl, Resource> nextLevel = new HashMap<QueryImpl, Resource>();
+    nextLevel.put(this, null);
+
+    while (!nextLevel.isEmpty()) {
+      Map<QueryImpl, Resource> current = nextLevel;
+      Map<Resource.Type, Set<Resource>> resourceMap = new HashMap<Resource.Type, Set<Resource>>();
+      Map<Resource.Type, Predicate> predicateMap = new HashMap<Resource.Type, Predicate>();
+
+      Map<Resource.Type, Request> requestMap = new HashMap<Resource.Type, Request>();
+
+      //All resource instances in tree have same set of property ids, only resource type is valuable
+      //Request and predicate are formed using data described above
+      for (QueryImpl query : current.keySet()) {
+
+        Resource.Type type = query.getResourceType();
+        if (!resourceMap.containsKey(type)) {
+          resourceMap.put(type, new HashSet<Resource>());
+          predicateMap.put(type, query.getPredicate());
+          requestMap.put(type, query.createRequest());
+        }
+
+        resourceMap.get(type).addAll(query.getProviderResources());
+
+      }
+
+      getClusterController().populateResources(resourceMap, requestMap, predicateMap);
+
+      //fill next level
+      nextLevel = new HashMap<QueryImpl, Resource>();
+
+      for (Map.Entry<QueryImpl, Resource> entry : current.entrySet()) {
+        QueryImpl query = entry.getKey();
+        Resource parent = entry.getValue();
+
+        Map<QueryImpl, Resource> subQueries = query.getSubQueries();
+
+        Collection<Resource> validResources = new ArrayList<Resource>();
+
+        for (Resource resource : query.getResourceIterable()) {
+          validResources.add(resource);
+        }
+
+        resourceTree.put(parent, validResources);
+
+        resourceNames.put(parent, query.getResultName());
+
+        nextLevel.putAll(subQueries);
+      }
 
-    if (LOG.isDebugEnabled()) {
-      //todo: include predicate info.  Need to implement toString for all predicates.
-      LOG.debug("Executing resource query: " + m_resource.getIds());
     }
 
-    Predicate predicate = createPredicate(m_resource);
-    Iterable<Resource> iterResource;
+    //create Result
+    Result result = createResult();
+    populateResult(result, resourceTree, resourceNames, null, null);
+    return result;
+  }
+
+  public String getResultName() {
+    return m_resource.isCollectionResource() ? m_resource.getResourceDefinition().getPluralName() :
+        m_resource.getResourceDefinition().getSingularName();
+  }
+
+  public Resource.Type getResourceType() {
+    return m_resource.getResourceDefinition().getType();
+  }
 
-    if (m_pageRequest == null) {
-      iterResource = getClusterController().getResources(
-          resourceType, createRequest(), predicate);
+  protected void populateResult(Result result, Map<Resource, Collection<Resource>> resourceTree,
+                                  Map<Resource, String> resourceNames,
+                                  TreeNode<Resource> node, Resource parent) {
+
+    TreeNode<Resource> tree;
+    if (node == null) {
+      tree = result.getResultTree();
+      if (m_resource.getIds().get(getResourceType()) == null) {
+        tree.setProperty("isCollection", "true");
+      }
     } else {
-      PageResponse pageResponse = getClusterController().getResources(
-          resourceType, createRequest(), predicate, m_pageRequest);
-      iterResource = pageResponse.getIterable();
+      tree = new TreeNodeImpl<Resource>(null, null, null);
+    }
+
+    if (node != null) {
+      tree.setProperty("isCollection", "false");
+      tree.setName(resourceNames.get(parent));
+      node.addChild(tree);
     }
 
-    TreeNode<Resource> tree = result.getResultTree();
     int count = 1;
-    for (Resource resource : iterResource) {
-      // add a child node for the resource and provide a unique name.  The name is never used.
-      //todo: provide a more meaningful node name
-      TreeNode<Resource> node = tree.addChild(resource, resource.getType() + ":" + count++);
-      for (Map.Entry<String, ResourceInstance> entry : m_mapSubResources.entrySet()) {
-        String subResCategory = entry.getKey();
-        ResourceInstance r = entry.getValue();
-
-        setParentIdsOnSubResource(resource, r);
-
-        TreeNode<Resource> childResult = r.getQuery().execute().getResultTree();
-        childResult.setName(subResCategory);
-        childResult.setProperty("isCollection", "false");
-        node.addChild(childResult);
+    for (Resource resource : resourceTree.get(parent)) {
+      TreeNode<Resource> subNode = tree.addChild(resource, resource.getType() + ":" + count++);
+      if (resourceTree.containsKey(resource)) {
+        populateResult(result, resourceTree, resourceNames, subNode, resource);
       }
     }
-    return result;
+
   }
 
+  public Map<QueryImpl, Resource> getSubQueries()
+      throws NoSuchParentResourceException, UnsupportedPropertyException, NoSuchResourceException, SystemException {
+    return getSubQueries(getResourceIterable());
+  }
+
+  protected Map<QueryImpl, Resource> getSubQueries(Iterable<Resource> resourceIterable) {
+    Map<QueryImpl, Resource> parentMap = new HashMap<QueryImpl, Resource>();
+
+    for (Resource parentResource : resourceIterable) {
+      //Need to copy Resource Instance because of delayed query execution
+      for (ResourceInstance resourceInstanceTemplate : m_mapSubResources.values()) {
+        ResourceInstance resourceInstance;
+        resourceInstance = resourceInstanceTemplate.createCopy();
+        setParentIdsOnSubResource(parentResource, resourceInstance);
+        parentMap.put((QueryImpl) resourceInstance.getQuery(), parentResource);
+
+      }
+    }
+
+    return parentMap;
+  }
+
+
   @Override
   public Predicate getPredicate() {
-    //todo: create predicate once
-    return createPredicate(m_resource);
+    if (m_predicate == null) {
+      m_predicate = createPredicate(m_resource);
+    }
+    return m_predicate;
   }
 
   @Override
@@ -301,7 +432,7 @@ public class QueryImpl implements Query {
     return predicate;
   }
 
-  private Request createRequest() {
+  public Request createRequest() {
     Set<String> setProperties = new HashSet<String>();
 
     Map<String, TemporalInfo> mapTemporalInfo    = new HashMap<String, TemporalInfo>();
@@ -346,29 +477,18 @@ public class QueryImpl implements Query {
     return new ResultImpl(true);
   }
 
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) return true;
-    if (o == null || getClass() != o.getClass()) return false;
-
-    QueryImpl that = (QueryImpl) o;
-
-    return m_mapCategoryTemporalInfo.equals(that.m_mapCategoryTemporalInfo) &&
-           m_mapPropertyTemporalInfo.equals(that.m_mapPropertyTemporalInfo) &&
-           m_setQueryProperties.equals(that.m_setQueryProperties) &&
-           m_mapSubResources.equals(that.m_mapSubResources) &&
-           m_resource.equals(that.m_resource) &&
-           m_userPredicate == null ? that.m_userPredicate == null : m_userPredicate.equals(that.m_userPredicate);
+  /**
+   * Copy template fields to given query
+   * @param query query to which fields will be copied
+   */
+  public void cloneFields(QueryImpl query) {
+    query.m_predicate = m_predicate;
+    query.m_mapCategoryTemporalInfo = m_mapCategoryTemporalInfo;
+    query.m_mapPropertyTemporalInfo = m_mapPropertyTemporalInfo;
+    query.m_mapSubResources = m_mapSubResources;
+    query.m_setQueryProperties = m_setQueryProperties;
+    query.m_pageRequest = m_pageRequest;
+    query.m_userPredicate = m_userPredicate;
   }
 
-  @Override
-  public int hashCode() {
-    int result = m_resource.hashCode();
-    result = 31 * result + m_setQueryProperties.hashCode();
-    result = 31 * result + m_mapPropertyTemporalInfo.hashCode();
-    result = 31 * result + m_mapCategoryTemporalInfo.hashCode();
-    result = 31 * result + m_mapSubResources.hashCode();
-    result = 31 * result + (m_userPredicate != null ? m_userPredicate.hashCode() : 0);
-    return result;
-  }
 }

ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstance.java

@@ -74,4 +74,10 @@ public interface ResourceInstance {
    * @return true if the resource is a collection resource; false otherwise
    */
   public boolean isCollectionResource();
+
+  /**
+   * Create clear copy of current resource instance
+   * @return new ResourceInstance
+   */
+  ResourceInstance createCopy();
 }

ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceImpl.java

@@ -132,25 +132,11 @@ public class ResourceInstanceImpl implements ResourceInstance {
   }
 
   @Override
-  public boolean equals(Object o) {
-    if (this == o) return true;
-    if (o == null || getClass() != o.getClass()) return false;
+  public ResourceInstance createCopy() {
+    ResourceInstanceImpl resourceInstance = new ResourceInstanceImpl(m_mapResourceIds, m_resourceDefinition, m_resourceFactory);
+    resourceInstance.m_mapSubResources = m_mapSubResources;
+    ((QueryImpl) m_query).cloneFields((QueryImpl) resourceInstance.m_query);
 
-    ResourceInstanceImpl that = (ResourceInstanceImpl) o;
-
-    return m_mapResourceIds.equals(that.m_mapResourceIds) &&
-           m_query == that.m_query &&
-           m_resourceDefinition.equals(that.m_resourceDefinition) &&
-           m_mapSubResources == null ? that.m_mapSubResources == null :
-               m_mapSubResources.equals(that.m_mapSubResources);
-  }
-
-  @Override
-  public int hashCode() {
-    int result = 13;
-    result = 31 * result + m_mapResourceIds.hashCode();
-    result = 31 * result + m_resourceDefinition.hashCode();
-    result = 31 * result + (m_mapSubResources != null ? m_mapSubResources.hashCode() : 0);
-    return result;
+    return resourceInstance;
   }
 }

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterControllerImpl.java

@@ -91,28 +91,22 @@ public class ClusterControllerImpl implements ClusterController {
 
   // ----- ClusterController -------------------------------------------------
 
-  @Override
   public Iterable<Resource> getResources(Resource.Type type, Request request, Predicate predicate)
       throws UnsupportedPropertyException,
              SystemException,
              NoSuchParentResourceException,
              NoSuchResourceException {
-    PageResponse response = getResources(type, request, predicate, null);
-    return response.getIterable();
+    return getResources(type, request, predicate, null).getIterable();
   }
 
   @Override
-  public PageResponse getResources(Resource.Type type, Request request, Predicate predicate, PageRequest pageRequest)
-      throws UnsupportedPropertyException,
-             SystemException,
-             NoSuchResourceException,
-             NoSuchParentResourceException {
-    ResourceProvider provider = ensureResourceProvider(type);
-    ensurePropertyProviders(type);
+  public Set<Resource> getRawResources(Resource.Type type, Request request, Predicate predicate)
+      throws UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException, SystemException {
+
     Set<Resource> resources;
 
-    ResourcePredicateEvaluator evaluator = provider instanceof ResourcePredicateEvaluator ?
-      (ResourcePredicateEvaluator) provider : DEFAULT_RESOURCE_PREDICATE_EVALUATOR;
+    ResourceProvider provider = ensureResourceProvider(type);
+    ensurePropertyProviders(type);
 
     if (provider == null) {
       resources = Collections.emptySet();
@@ -125,8 +119,32 @@ public class ClusterControllerImpl implements ClusterController {
       }
       checkProperties(type, request, predicate);
 
-      Set<Resource> providerResources = provider.getResources(request, predicate);
-      providerResources               = populateResources(type, providerResources, request, predicate);
+      resources = provider.getResources(request, predicate);
+    }
+    return resources;
+  }
+
+  @Override
+  public Iterable<Resource> getResources(Resource.Type type, Set<Resource> providerResources, Predicate predicate)
+      throws NoSuchParentResourceException, UnsupportedPropertyException, NoSuchResourceException, SystemException {
+    return getResources(type, providerResources, predicate, null).getIterable();
+  }
+
+  @Override
+  public PageResponse getResources(Resource.Type type, Set<Resource> providerResources, Predicate predicate, PageRequest pageRequest)
+      throws UnsupportedPropertyException,
+      SystemException,
+      NoSuchResourceException,
+      NoSuchParentResourceException {
+
+    Set<Resource> resources;
+    ResourceProvider provider = ensureResourceProvider(type);
+
+    ResourcePredicateEvaluator evaluator = provider instanceof ResourcePredicateEvaluator ?
+        (ResourcePredicateEvaluator) provider : DEFAULT_RESOURCE_PREDICATE_EVALUATOR;
+
+    if (!providerResources.isEmpty()) {
+
 
       Comparator<Resource> resourceComparator = pageRequest == null || pageRequest.getComparator() == null ?
           comparator : pageRequest.getComparator();
@@ -154,12 +172,25 @@ public class ClusterControllerImpl implements ClusterController {
         }
       }
       resources = sortedResources;
+    } else {
+      resources = providerResources;
     }
 
     return new PageResponseImpl(new ResourceIterable(resources, predicate, evaluator),
       0, null, null);
   }
 
+  public PageResponse getResources(Resource.Type type, Request request, Predicate predicate, PageRequest pageRequest)
+      throws UnsupportedPropertyException,
+             SystemException,
+             NoSuchResourceException,
+             NoSuchParentResourceException {
+
+    Set<Resource> providerResources = getRawResources(type, request, predicate);
+    populateResources(type, providerResources, request, predicate);
+    return getResources(type, providerResources, predicate, pageRequest);
+  }
+
   @Override
   public Schema getSchema(Resource.Type type) {
     Schema schema;
@@ -353,10 +384,11 @@ public class ClusterControllerImpl implements ClusterController {
    *
    * @throws SystemException if unable to populate the resources
    */
-  private Set<Resource> populateResources(Resource.Type type,
-                                          Set<Resource> resources,
-                                          Request request,
-                                          Predicate predicate) throws SystemException {
+  @Override
+  public Set<Resource> populateResources(Resource.Type type,
+                                         Set<Resource> resources,
+                                         Request request,
+                                         Predicate predicate) throws SystemException {
     Set<Resource> keepers = resources;
 
     for (PropertyProvider propertyProvider : propertyProviders.get(type)) {
@@ -367,6 +399,30 @@ public class ClusterControllerImpl implements ClusterController {
     return keepers;
   }
 
+  /**
+   * Performs bulk population of the given resources from the associated property providers.  This
+   * method may filter the resources based on the predicate and return a subset
+   * of the given resources.
+   * @param resourceMap resources grouped by type
+   * @param requestMap type-request map
+   * @param predicateMap type-predicate map
+   * @return the set of resources that were successfully populated grouped by resource type
+   * @throws SystemException
+   */
+  @Override
+  public Map<Resource.Type, Set<Resource>> populateResources(Map<Resource.Type, Set<Resource>> resourceMap,
+                                Map<Resource.Type, Request> requestMap,
+                                Map<Resource.Type, Predicate> predicateMap) throws SystemException {
+
+    Map<Resource.Type, Set<Resource>> keepers = new HashMap<Resource.Type, Set<Resource>>();
+
+    for (Resource.Type type : resourceMap.keySet()) {
+      keepers.put(type, populateResources(type, resourceMap.get(type), requestMap.get(type), predicateMap.get(type)));
+    }
+
+    return keepers;
+  }
+
   /**
    * Indicates whether or not the given property provider can service the given request.
    *

ambari-server/src/main/java/org/apache/ambari/server/controller/spi/ClusterController.java

@@ -18,6 +18,9 @@
 package org.apache.ambari.server.controller.spi;
 
 
+import java.util.Map;
+import java.util.Set;
+
 /**
  * The cluster controller is the main access point for accessing resources
  * from the backend sources.  A cluster controller maintains a mapping of
@@ -27,55 +30,6 @@ public interface ClusterController {
 
   // ----- Monitoring ------------------------------------------------------
 
-  /**
-   * Get the resources of the given type filtered by the given request and
-   * predicate objects.
-   *
-   * @param type      the type of the requested resources
-   * @param request   the request object which defines the desired set of properties
-   * @param predicate the predicate object which filters which resources are returned
-   *
-   * @return an iterable object of the requested resources
-   *
-   * @throws UnsupportedPropertyException thrown if the request or predicate contain
-   *                                      unsupported property ids
-   * @throws SystemException an internal exception occurred
-   * @throws NoSuchResourceException no matching resource(s) found
-   * @throws NoSuchParentResourceException a specified parent resource doesn't exist
-   */
-  public Iterable<Resource> getResources(Resource.Type type,
-                                         Request request,
-                                         Predicate predicate)
-      throws UnsupportedPropertyException,
-             SystemException,
-             NoSuchResourceException,
-             NoSuchParentResourceException;
-
-  /**
-   * Get a page of resources of the given type filtered by the given request,
-   * predicate objects and page request.
-   *
-   * @param type        the type of the requested resources
-   * @param request     the request object which defines the desired set of properties
-   * @param predicate   the predicate object which filters which resources are returned
-   * @param pageRequest the page request for a paginated response
-   *
-   * @return a page response representing the requested page of resources
-   *
-   * @throws UnsupportedPropertyException thrown if the request or predicate contain
-   *                                      unsupported property ids
-   * @throws SystemException an internal exception occurred
-   * @throws NoSuchResourceException no matching resource(s) found
-   * @throws NoSuchParentResourceException a specified parent resource doesn't exist
-   */
-  public PageResponse getResources(Resource.Type type,
-                                         Request request,
-                                         Predicate predicate,
-                                         PageRequest pageRequest)
-      throws UnsupportedPropertyException,
-      SystemException,
-      NoSuchResourceException,
-      NoSuchParentResourceException;
 
   /**
    * Get the {@link Schema schema} for the given resource type.  The schema
@@ -152,4 +106,103 @@ public interface ClusterController {
              SystemException,
              NoSuchResourceException,
              NoSuchParentResourceException ;
+
+  /**
+   * Get the not populated resources of the given type filtered by the given request and
+   * predicate objects.
+   *
+   * @param type      the type of the requested resources
+   * @param request   the request object which defines the desired set of properties
+   * @param predicate the predicate object which filters which resources are returned
+   *
+   * @return an iterable object of the requested resources
+   *
+   * @throws UnsupportedPropertyException thrown if the request or predicate contain
+   *                                      unsupported property ids
+   * @throws SystemException an internal exception occurred
+   * @throws NoSuchResourceException no matching resource(s) found
+   * @throws NoSuchParentResourceException a specified parent resource doesn't exist
+   */
+  Set<Resource> getRawResources(Resource.Type type, Request request, Predicate predicate)
+      throws UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException, SystemException;
+
+  /**
+   * Get a page of resources of the given type filtered by the given request,
+   * predicate objects and page request.
+   *
+   *
+   * @param type type of resources
+   * @param providerResources set of populated Resources
+   * @param predicate   the predicate object which filters which resources are returned
+   * @param pageRequest the page request for a paginated response
+   *
+   * @return a page response representing the requested page of resources
+   *
+   * @throws UnsupportedPropertyException thrown if the request or predicate contain
+   *                                      unsupported property ids
+   * @throws SystemException an internal exception occurred
+   * @throws NoSuchResourceException no matching resource(s) found
+   * @throws NoSuchParentResourceException a specified parent resource doesn't exist
+   */
+  PageResponse getResources(Resource.Type type, Set<Resource> providerResources, Predicate predicate, PageRequest pageRequest)
+      throws UnsupportedPropertyException,
+      SystemException,
+      NoSuchResourceException,
+      NoSuchParentResourceException;
+
+  /**
+   * Get a page of resources of the given type filtered by the given request,
+   * predicate objects and page request.
+   *
+   *
+   * @param type type of resources
+   * @param providerResources set of populated Resources
+   * @param predicate   the predicate object which filters which resources are returned
+   *
+   * @return a page response representing the requested page of resources
+   *
+   * @throws UnsupportedPropertyException thrown if the request or predicate contain
+   *                                      unsupported property ids
+   * @throws SystemException an internal exception occurred
+   * @throws NoSuchResourceException no matching resource(s) found
+   * @throws NoSuchParentResourceException a specified parent resource doesn't exist
+   */
+  Iterable<Resource> getResources(Resource.Type type, Set<Resource> providerResources, Predicate predicate)
+  throws NoSuchParentResourceException, UnsupportedPropertyException, NoSuchResourceException, SystemException;
+
+  /**
+   * Populate the given resources from the associated property providers.  This
+   * method may filter the resources based on the predicate and return a subset
+   * of the given resources.
+   *
+   * @param type       the resource type
+   * @param resources  the resources to be populated
+   * @param request    the request
+   * @param predicate  the predicate
+   *
+   * @return the set of resources that were successfully populated
+   *
+   * @throws SystemException if unable to populate the resources
+   */
+  Set<Resource> populateResources(Resource.Type type,
+                                  Set<Resource> resources,
+                                  Request request,
+                                  Predicate predicate) throws SystemException;
+
+  /**
+   * Performs bulk population of the given resources from the associated property providers.  This
+   * method may filter the resources based on the predicate and return a subset
+   * of the given resources.
+   *
+   * @param resourceMap resources grouped by type
+   * @param requestMap type-request map
+   * @param predicateMap type-predicate map
+   *
+   * @return the set of resources that were successfully populated grouped by resource type
+   *
+   * @throws SystemException
+   */
+  Map<Resource.Type, Set<Resource>> populateResources(Map<Resource.Type, Set<Resource>> resourceMap,
+                         Map<Resource.Type, Request> requestMap,
+                         Map<Resource.Type, Predicate> predicateMap) throws SystemException;
 }

ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java

@@ -55,12 +55,10 @@ public class QueryImplTest {
     Schema componentSchema = createNiceMock(Schema.class);
     Resource componentResource = createNiceMock(Resource.class);
     String componentPropertyId = "componentId";
-    Query hostComponentQuery = createStrictMock(Query.class);
-    Result hostComponentQueryResult = createNiceMock(Result.class);
+    QueryImpl hostComponentQuery = createStrictMock(QueryImpl.class);
 
     TreeNode<Resource> tree = new TreeNodeImpl<Resource>(null, null, null);
-    TreeNode<Resource> hostComponentResultNode = new TreeNodeImpl<Resource>(null, null, null);
-    List<Resource> listResources = Collections.singletonList(componentResource);
+    Set<Resource> listResources = Collections.singleton(componentResource);
 
     Map<Resource.Type, String> mapResourceIds = new HashMap<Resource.Type, String>();
     mapResourceIds.put(Resource.Type.Cluster, "clusterName");
@@ -80,6 +78,8 @@ public class QueryImplTest {
     expect(componentResourceInstance.getSubResources()).andReturn(mapChildren).anyTimes();
     expect(componentResourceInstance.getIds()).andReturn(mapResourceIds).anyTimes();
 
+    expect(hostResourceInstance.createCopy()).andReturn(hostResourceInstance).anyTimes();
+
     expect(componentResourceDefinition.getType()).andReturn(Resource.Type.Component).anyTimes();
 
     expect(componentResource.getType()).andReturn(Resource.Type.Component).anyTimes();
@@ -91,9 +91,12 @@ public class QueryImplTest {
     expect(componentSchema.getKeyPropertyId(Resource.Type.Service)).andReturn("serviceId");
     expect(componentSchema.getKeyPropertyId(Resource.Type.Component)).andReturn(componentPropertyId).atLeastOnce();
 
-    expect(m_controller.getResources(eq(Resource.Type.Component), eq(PropertyHelper.getReadRequest(Collections.<String>emptySet())),
+    expect(m_controller.getRawResources(eq(Resource.Type.Component), eq(PropertyHelper.getReadRequest(Collections.<String>emptySet())),
         eq(predicate))).andReturn(listResources);
 
+    expect(m_controller.getResources(eq(Resource.Type.Component), eq(listResources), eq(predicate))).andReturn(
+        listResources);
+
     expect(result.getResultTree()).andReturn(tree).anyTimes();
 
     Map<Resource.Type, String> mapResourceIdsSet = new HashMap<Resource.Type, String>(mapResourceIds);
@@ -103,25 +106,32 @@ public class QueryImplTest {
     expect(hostResourceInstance.getQuery()).andReturn(hostComponentQuery).anyTimes();
 
     expect(hostResourceDefinition.getType()).andReturn(Resource.Type.Host);
-    expect(hostComponentQuery.execute()).andReturn(hostComponentQueryResult);
-    expect(hostComponentQueryResult.getResultTree()).andReturn(hostComponentResultNode);
+
+//    expect(hostComponentQuery.getResourceInstance()).andReturn(hostResourceInstance);
+    expect(hostComponentQuery.getResourceType()).andReturn(Resource.Type.Host);
+    expect(hostComponentQuery.getPredicate()).andReturn(predicate);
+    expect(hostComponentQuery.createRequest()).andReturn(PropertyHelper.getReadRequest(Collections.<String>emptySet()));
+    expect(hostComponentQuery.getProviderResources()).andReturn(Collections.<Resource>emptySet());
+    expect(hostComponentQuery.getSubQueries()).andReturn(Collections.<QueryImpl, Resource>emptyMap());
+    expect(hostComponentQuery.getResourceIterable()).andReturn(Collections.<Resource>emptySet());
+    expect(hostComponentQuery.getResultName()).andReturn("host_components");
+
 
     replay(m_controller, result, componentResourceInstance, componentResourceDefinition, hostResourceInstance, componentSchema, componentResource,
-        hostComponentQuery, hostComponentQueryResult);
+        hostComponentQuery);
 
     QueryImpl query = new TestQuery(componentResourceInstance, result);
     query.execute();
 
     verify(m_controller, result, componentResourceInstance, componentResourceDefinition, hostResourceInstance, componentSchema, componentResource,
-        hostComponentQuery, hostComponentQueryResult);
+        hostComponentQuery);
 
     assertEquals(1, tree.getChildren().size());
     TreeNode<Resource> componentNode = tree.getChild("Component:1");
     assertEquals("Component:1", componentNode.getName());
     assertEquals(componentResource, componentNode.getObject());
     assertEquals(1, componentNode.getChildren().size());
-    assertSame(hostComponentResultNode, componentNode.getChild("host_components"));
-    assertEquals("false", hostComponentResultNode.getProperty("isCollection"));
+    assertEquals("false", componentNode.getChild("host_components").getProperty("isCollection"));
   }
 
   @Test
@@ -141,7 +151,7 @@ public class QueryImplTest {
     setPropertyIds.add(componentPropertyId);
 
     TreeNode<Resource> tree = new TreeNodeImpl<Resource>(null, null, null);
-    List<Resource> listResources = Collections.singletonList(componentResource);
+    Set<Resource> resourceSet = Collections.singleton(componentResource);
 
     Map<Resource.Type, String> mapResourceIds = new HashMap<Resource.Type, String>();
     mapResourceIds.put(Resource.Type.Cluster, "clusterName");
@@ -168,8 +178,12 @@ public class QueryImplTest {
 
     expect(result.getResultTree()).andReturn(tree).anyTimes();
 
-    expect(m_controller.getResources(eq(Resource.Type.Component), eq(PropertyHelper.getReadRequest(setPropertyIds)),
-        eq(predicate))).andReturn(listResources);
+
+    expect(m_controller.getRawResources(eq(Resource.Type.Component), eq(PropertyHelper.getReadRequest(setPropertyIds)),
+        eq(predicate))).andReturn(resourceSet);
+
+    expect(m_controller.getResources(eq(Resource.Type.Component), eq(resourceSet), eq(predicate))).andReturn(
+        resourceSet);
 
     expect(componentResourceInstance.getSubResources()).andReturn(Collections.<String, ResourceInstance>emptyMap()).anyTimes();
 
@@ -197,7 +211,7 @@ public class QueryImplTest {
     String clusterPropertyId = "clusterId";
 
     TreeNode<Resource> tree = new TreeNodeImpl<Resource>(null, null, null);
-    List<Resource> listResources = Collections.singletonList(clusterResource);
+    Set<Resource> resourceSet = Collections.singleton(clusterResource);
 
     Map<Resource.Type, String> mapResourceIds = new HashMap<Resource.Type, String>();
 
@@ -215,8 +229,11 @@ public class QueryImplTest {
 
     expect(result.getResultTree()).andReturn(tree).atLeastOnce();
 
-    expect(m_controller.getResources(eq(Resource.Type.Component), eq(PropertyHelper.getReadRequest(Collections.singleton(clusterPropertyId))),
-        (Predicate) isNull())).andReturn(listResources);
+    expect(m_controller.getRawResources(eq(Resource.Type.Component), eq(PropertyHelper.getReadRequest(Collections.singleton(clusterPropertyId))),
+        isNull(Predicate.class))).andReturn(resourceSet);
+
+    expect(m_controller.getResources(eq(Resource.Type.Component), eq(resourceSet), isNull(Predicate.class))).andReturn(
+        resourceSet);
 
 
     expect(clusterResourceInstance.getSubResources()).andReturn(Collections.<String, ResourceInstance>emptyMap()).anyTimes();
@@ -247,7 +264,7 @@ public class QueryImplTest {
     Predicate userPredicate = createNiceMock(Predicate.class);
 
     TreeNode<Resource> tree = new TreeNodeImpl<Resource>(null, null, null);
-    List<Resource> listResources = Collections.singletonList(clusterResource);
+    Set<Resource> resourceSet = Collections.singleton(clusterResource);
 
     Map<Resource.Type, String> mapResourceIds = new HashMap<Resource.Type, String>();
 
@@ -264,8 +281,14 @@ public class QueryImplTest {
 
     expect(result.getResultTree()).andReturn(tree).anyTimes();
 
-    expect(m_controller.getResources(eq(Resource.Type.Component), eq(PropertyHelper.getReadRequest(Collections.singleton(clusterPropertyId))),
-        eq(userPredicate))).andReturn(listResources);
+//    expect(m_controller.getResources(eq(Resource.Type.Component), eq(PropertyHelper.getReadRequest(Collections.singleton(clusterPropertyId))),
+//        eq(userPredicate))).andReturn(resourceSet);
+
+    expect(m_controller.getRawResources(eq(Resource.Type.Component), eq(PropertyHelper.getReadRequest(Collections.singleton(clusterPropertyId))),
+        eq(userPredicate))).andReturn(resourceSet);
+
+    expect(m_controller.getResources(eq(Resource.Type.Component), eq(resourceSet), eq(userPredicate))).andReturn(
+        resourceSet);
 
     expect(clusterResourceInstance.getSubResources()).andReturn(Collections.<String, ResourceInstance>emptyMap()).anyTimes();
 
@@ -301,7 +324,7 @@ public class QueryImplTest {
     setPropertyIds.add(componentPropertyId);
 
     TreeNode<Resource> tree = new TreeNodeImpl<Resource>(null, null, null);
-    List<Resource> listResources = Collections.singletonList(componentResource);
+    Set<Resource> resourceSet = Collections.singleton(componentResource);
 
     Map<Resource.Type, String> mapResourceIds = new HashMap<Resource.Type, String>();
     mapResourceIds.put(Resource.Type.Cluster, "clusterName");
@@ -333,8 +356,14 @@ public class QueryImplTest {
 
     expect(result.getResultTree()).andReturn(tree).anyTimes();
 
-    expect(m_controller.getResources(eq(Resource.Type.Component), eq(PropertyHelper.getReadRequest(setPropertyIds)),
-        eq(predicate))).andReturn(listResources);
+//    expect(m_controller.getResources(eq(Resource.Type.Component), eq(PropertyHelper.getReadRequest(setPropertyIds)),
+//        eq(predicate))).andReturn(resourceSet);
+
+    expect(m_controller.getRawResources(eq(Resource.Type.Component), eq(PropertyHelper.getReadRequest(setPropertyIds)),
+        eq(predicate))).andReturn(resourceSet);
+
+    expect(m_controller.getResources(eq(Resource.Type.Component), eq(resourceSet), eq(predicate))).andReturn(
+        resourceSet);
 
     expect(componentResourceInstance.getSubResources()).andReturn(Collections.<String, ResourceInstance>emptyMap()).anyTimes();
 

ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterControllerImplTest.java

@@ -103,7 +103,7 @@ public class ClusterControllerImplTest {
 
   @Test
   public void testGetResources() throws Exception{
-    ClusterController controller = new ClusterControllerImpl(new TestProviderModule());
+    ClusterControllerImpl controller = new ClusterControllerImpl(new TestProviderModule());
 
     Set<String> propertyIds = new HashSet<String>();
 
@@ -124,7 +124,7 @@ public class ClusterControllerImplTest {
 
   @Test
   public void testGetResourcesPageFromStart() throws Exception{
-    ClusterController controller = new ClusterControllerImpl(new TestProviderModule());
+    ClusterControllerImpl controller = new ClusterControllerImpl(new TestProviderModule());
 
     Set<String> propertyIds = new HashSet<String>();
 
@@ -167,7 +167,7 @@ public class ClusterControllerImplTest {
 
   @Test
   public void testGetResourcesPageFromOffset() throws Exception{
-    ClusterController controller = new ClusterControllerImpl(new TestProviderModule());
+    ClusterControllerImpl controller = new ClusterControllerImpl(new TestProviderModule());
 
     Set<String> propertyIds = new HashSet<String>();
 
@@ -218,7 +218,7 @@ public class ClusterControllerImplTest {
 
   @Test
   public void testGetResourcesPageToEnd() throws Exception{
-    ClusterController controller = new ClusterControllerImpl(new TestProviderModule());
+    ClusterControllerImpl controller = new ClusterControllerImpl(new TestProviderModule());
 
     Set<String> propertyIds = new HashSet<String>();
 
@@ -261,7 +261,7 @@ public class ClusterControllerImplTest {
 
   @Test
   public void testGetResourcesPageToOffset() throws Exception{
-    ClusterController controller = new ClusterControllerImpl(new TestProviderModule());
+    ClusterControllerImpl controller = new ClusterControllerImpl(new TestProviderModule());
 
     Set<String> propertyIds = new HashSet<String>();
 
@@ -312,7 +312,7 @@ public class ClusterControllerImplTest {
 
   @Test
   public void testGetResourcesEmptyRequest() throws Exception{
-    ClusterController controller = new ClusterControllerImpl(new TestProviderModule());
+    ClusterControllerImpl controller = new ClusterControllerImpl(new TestProviderModule());
 
     Set<String> propertyIds = new HashSet<String>();
 
@@ -330,7 +330,7 @@ public class ClusterControllerImplTest {
 
   @Test
   public void testGetResourcesCheckOrder() throws Exception{
-    ClusterController controller = new ClusterControllerImpl(new TestProviderModule());
+    ClusterControllerImpl controller = new ClusterControllerImpl(new TestProviderModule());
 
     Set<String> propertyIds = new HashSet<String>();
 
@@ -356,7 +356,7 @@ public class ClusterControllerImplTest {
 
   @Test
   public void testGetResourcesWithPredicate() throws Exception{
-    ClusterController controller = new ClusterControllerImpl(new TestProviderModule());
+    ClusterControllerImpl controller = new ClusterControllerImpl(new TestProviderModule());
 
     Set<String> propertyIds = new HashSet<String>();
 
@@ -381,7 +381,7 @@ public class ClusterControllerImplTest {
 
   @Test
   public void testGetResourcesWithUnsupportedPropertyPredicate() throws Exception{
-    ClusterController controller = new ClusterControllerImpl(new TestProviderModule());
+    ClusterControllerImpl controller = new ClusterControllerImpl(new TestProviderModule());
 
     Set<String> propertyIds = new HashSet<String>();
 
@@ -404,7 +404,7 @@ public class ClusterControllerImplTest {
 
   @Test
   public void testGetResourcesWithUnsupportedPropertyRequest() throws Exception{
-    ClusterController controller = new ClusterControllerImpl(new TestProviderModule());
+    ClusterControllerImpl controller = new ClusterControllerImpl(new TestProviderModule());
 
     Set<String> propertyIds = new HashSet<String>();