浏览代码

Merge -c 1488806 from trunk to branch-2 to fix YARN-749. Rename ResourceRequest.(get,set)HostName to ResourceRequest.(get,set)ResourceName. Contributed by Arun C. Murthy.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1488807 13f79535-47bb-0310-9956-ffa450edef68
Arun Murthy 12 年之前
父节点
当前提交
7246cf9b41
共有 14 个文件被更改,包括 52 次插入45 次删除
  1. 2 2
      hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
  2. 1 1
      hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
  3. 1 1
      hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
  4. 3 0
      hadoop-yarn-project/CHANGES.txt
  5. 20 16
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
  6. 8 8
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
  7. 1 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
  8. 4 4
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
  9. 6 6
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
  10. 2 2
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
  11. 1 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
  12. 1 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
  13. 1 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
  14. 1 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java

+ 2 - 2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java

@@ -241,7 +241,7 @@ public abstract class RMContainerRequestor extends RMCommunicator {
               // specified for the blacklisted host to be 0.
               // specified for the blacklisted host to be 0.
               ResourceRequest zeroedRequest =
               ResourceRequest zeroedRequest =
                   ResourceRequest.newInstance(req.getPriority(),
                   ResourceRequest.newInstance(req.getPriority(),
-                    req.getHostName(), req.getCapability(),
+                    req.getResourceName(), req.getCapability(),
                     req.getNumContainers());
                     req.getNumContainers());
 
 
               zeroedRequest.setNumContainers(0);
               zeroedRequest.setNumContainers(0);
@@ -323,7 +323,7 @@ public abstract class RMContainerRequestor extends RMCommunicator {
     if (remoteRequest == null) {
     if (remoteRequest == null) {
       remoteRequest = recordFactory.newRecordInstance(ResourceRequest.class);
       remoteRequest = recordFactory.newRecordInstance(ResourceRequest.class);
       remoteRequest.setPriority(priority);
       remoteRequest.setPriority(priority);
-      remoteRequest.setHostName(resourceName);
+      remoteRequest.setResourceName(resourceName);
       remoteRequest.setCapability(capability);
       remoteRequest.setCapability(capability);
       remoteRequest.setNumContainers(0);
       remoteRequest.setNumContainers(0);
       reqMap.put(capability, remoteRequest);
       reqMap.put(capability, remoteRequest);

+ 1 - 1
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java

@@ -227,7 +227,7 @@ public class MRAppBenchmark {
                 List<ResourceRequest> askList = request.getAskList();
                 List<ResourceRequest> askList = request.getAskList();
                 List<Container> containers = new ArrayList<Container>();
                 List<Container> containers = new ArrayList<Container>();
                 for (ResourceRequest req : askList) {
                 for (ResourceRequest req : askList) {
-                  if (!ResourceRequest.isAnyLocation(req.getHostName())) {
+                  if (!ResourceRequest.isAnyLocation(req.getResourceName())) {
                     continue;
                     continue;
                   }
                   }
                   int numContainers = req.getNumContainers();
                   int numContainers = req.getNumContainers();

+ 1 - 1
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java

@@ -1219,7 +1219,7 @@ public class TestRMContainerAllocator {
       List<ResourceRequest> askCopy = new ArrayList<ResourceRequest>();
       List<ResourceRequest> askCopy = new ArrayList<ResourceRequest>();
       for (ResourceRequest req : ask) {
       for (ResourceRequest req : ask) {
         ResourceRequest reqCopy = ResourceRequest.newInstance(req
         ResourceRequest reqCopy = ResourceRequest.newInstance(req
-            .getPriority(), req.getHostName(), req.getCapability(), req
+            .getPriority(), req.getResourceName(), req.getCapability(), req
             .getNumContainers());
             .getNumContainers());
         askCopy.add(reqCopy);
         askCopy.add(reqCopy);
       }
       }

+ 3 - 0
hadoop-yarn-project/CHANGES.txt

@@ -61,6 +61,9 @@ Release 2.1.0-beta - UNRELEASED
     YARN-735. Make ApplicationAttemptId, ContaienrId and NodeId immutable.
     YARN-735. Make ApplicationAttemptId, ContaienrId and NodeId immutable.
     (Jian He via sseth)
     (Jian He via sseth)
 
 
+    YARN-749. Rename ResourceRequest.(get,set)HostName to
+    ResourceRequest.(get,set)ResourceName. (acmurthy)
+
   NEW FEATURES
   NEW FEATURES
 
 
     YARN-482. FS: Extend SchedulingMode to intermediate queues. 
     YARN-482. FS: Extend SchedulingMode to intermediate queues. 

+ 20 - 16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java

@@ -57,7 +57,7 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest> {
       String hostName, Resource capability, int numContainers) {
       String hostName, Resource capability, int numContainers) {
     ResourceRequest request = Records.newRecord(ResourceRequest.class);
     ResourceRequest request = Records.newRecord(ResourceRequest.class);
     request.setPriority(priority);
     request.setPriority(priority);
-    request.setHostName(hostName);
+    request.setResourceName(hostName);
     request.setCapability(capability);
     request.setCapability(capability);
     request.setNumContainers(numContainers);
     request.setNumContainers(numContainers);
     return request;
     return request;
@@ -99,28 +99,32 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest> {
   public abstract void setPriority(Priority priority);
   public abstract void setPriority(Priority priority);
   
   
   /**
   /**
-   * Get the <em>host/rack</em> on which the allocation is desired.
+   * Get the resource (e.g. <em>host/rack</em>) on which the allocation 
+   * is desired.
    * 
    * 
-   * A special value of <em>*</em> signifies that <em>any</em> host/rack is 
-   * acceptable.
+   * A special value of <em>*</em> signifies that <em>any</em> resource 
+   * (host/rack) is acceptable.
    * 
    * 
-   * @return <em>host/rack</em> on which the allocation is desired
+   * @return resource (e.g. <em>host/rack</em>) on which the allocation 
+   *                  is desired
    */
    */
   @Public
   @Public
   @Stable
   @Stable
-  public abstract String getHostName();
+  public abstract String getResourceName();
 
 
   /**
   /**
-   * Set <em>host/rack</em> on which the allocation is desired.
+   * Set the resource (e.g. <em>host/rack</em>) on which the allocation 
+   * is desired.
    * 
    * 
-   * A special value of <em>*</em> signifies that <em>any</em> host/rack is 
-   * acceptable.
+   * A special value of <em>*</em> signifies that <em>any</em> resource 
+   * (e.g. host/rack) is acceptable. 
    * 
    * 
-   * @param hostName <em>host/rack</em> on which the allocation is desired
+   * @param resourceName (e.g. <em>host/rack</em>) on which the 
+   *                     allocation is desired
    */
    */
   @Public
   @Public
   @Stable
   @Stable
-  public abstract void setHostName(String hostName);
+  public abstract void setResourceName(String resourceName);
   
   
   /**
   /**
    * Get the <code>Resource</code> capability of the request.
    * Get the <code>Resource</code> capability of the request.
@@ -198,7 +202,7 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest> {
     final int prime = 2153;
     final int prime = 2153;
     int result = 2459;
     int result = 2459;
     Resource capability = getCapability();
     Resource capability = getCapability();
-    String hostName = getHostName();
+    String hostName = getResourceName();
     Priority priority = getPriority();
     Priority priority = getPriority();
     result =
     result =
         prime * result + ((capability == null) ? 0 : capability.hashCode());
         prime * result + ((capability == null) ? 0 : capability.hashCode());
@@ -223,11 +227,11 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest> {
         return false;
         return false;
     } else if (!capability.equals(other.getCapability()))
     } else if (!capability.equals(other.getCapability()))
       return false;
       return false;
-    String hostName = getHostName();
+    String hostName = getResourceName();
     if (hostName == null) {
     if (hostName == null) {
-      if (other.getHostName() != null)
+      if (other.getResourceName() != null)
         return false;
         return false;
-    } else if (!hostName.equals(other.getHostName()))
+    } else if (!hostName.equals(other.getResourceName()))
       return false;
       return false;
     if (getNumContainers() != other.getNumContainers())
     if (getNumContainers() != other.getNumContainers())
       return false;
       return false;
@@ -245,7 +249,7 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest> {
     int priorityComparison = this.getPriority().compareTo(other.getPriority());
     int priorityComparison = this.getPriority().compareTo(other.getPriority());
     if (priorityComparison == 0) {
     if (priorityComparison == 0) {
       int hostNameComparison =
       int hostNameComparison =
-          this.getHostName().compareTo(other.getHostName());
+          this.getResourceName().compareTo(other.getResourceName());
       if (hostNameComparison == 0) {
       if (hostNameComparison == 0) {
         int capabilityComparison =
         int capabilityComparison =
             this.getCapability().compareTo(other.getCapability());
             this.getCapability().compareTo(other.getCapability());

+ 8 - 8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java

@@ -98,22 +98,22 @@ public class ResourceRequestPBImpl extends  ResourceRequest {
     this.priority = priority;
     this.priority = priority;
   }
   }
   @Override
   @Override
-  public String getHostName() {
+  public String getResourceName() {
     ResourceRequestProtoOrBuilder p = viaProto ? proto : builder;
     ResourceRequestProtoOrBuilder p = viaProto ? proto : builder;
-    if (!p.hasHostName()) {
+    if (!p.hasResourceName()) {
       return null;
       return null;
     }
     }
-    return (p.getHostName());
+    return (p.getResourceName());
   }
   }
 
 
   @Override
   @Override
-  public void setHostName(String hostName) {
+  public void setResourceName(String resourceName) {
     maybeInitBuilder();
     maybeInitBuilder();
-    if (hostName == null) {
-      builder.clearHostName();
+    if (resourceName == null) {
+      builder.clearResourceName();
       return;
       return;
     }
     }
-    builder.setHostName((hostName));
+    builder.setResourceName((resourceName));
   }
   }
   @Override
   @Override
   public Resource getCapability() {
   public Resource getCapability() {
@@ -179,7 +179,7 @@ public class ResourceRequestPBImpl extends  ResourceRequest {
   public String toString() {
   public String toString() {
     return "{Priority: " + getPriority() + ", Capability: " + getCapability()
     return "{Priority: " + getPriority() + ", Capability: " + getCapability()
         + ", # Containers: " + getNumContainers()
         + ", # Containers: " + getNumContainers()
-        + ", Location: " + getHostName()
+        + ", Location: " + getResourceName()
         + ", Relax Locality: " + getRelaxLocality() + "}";
         + ", Relax Locality: " + getRelaxLocality() + "}";
   }
   }
 }
 }

+ 1 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto

@@ -201,7 +201,7 @@ message NodeReportProto {
 ////////////////////////////////////////////////////////////////////////
 ////////////////////////////////////////////////////////////////////////
 message ResourceRequestProto {
 message ResourceRequestProto {
   optional PriorityProto priority = 1;
   optional PriorityProto priority = 1;
-  optional string host_name = 2;
+  optional string resource_name = 2;
   optional ResourceProto capability = 3;
   optional ResourceProto capability = 3;
   optional int32 num_containers = 4;
   optional int32 num_containers = 4;
   optional bool relax_locality = 5 [default = true];
   optional bool relax_locality = 5 [default = true];

+ 4 - 4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java

@@ -97,8 +97,8 @@ public class BuilderUtils {
       // Compare priority, host and capability
       // Compare priority, host and capability
       int ret = r1.getPriority().compareTo(r2.getPriority());
       int ret = r1.getPriority().compareTo(r2.getPriority());
       if (ret == 0) {
       if (ret == 0) {
-        String h1 = r1.getHostName();
-        String h2 = r2.getHostName();
+        String h1 = r1.getResourceName();
+        String h2 = r2.getResourceName();
         ret = h1.compareTo(h2);
         ret = h1.compareTo(h2);
       }
       }
       if (ret == 0) {
       if (ret == 0) {
@@ -296,7 +296,7 @@ public class BuilderUtils {
     ResourceRequest request = recordFactory
     ResourceRequest request = recordFactory
         .newRecordInstance(ResourceRequest.class);
         .newRecordInstance(ResourceRequest.class);
     request.setPriority(priority);
     request.setPriority(priority);
-    request.setHostName(hostName);
+    request.setResourceName(hostName);
     request.setCapability(capability);
     request.setCapability(capability);
     request.setNumContainers(numContainers);
     request.setNumContainers(numContainers);
     return request;
     return request;
@@ -306,7 +306,7 @@ public class BuilderUtils {
     ResourceRequest request = recordFactory
     ResourceRequest request = recordFactory
         .newRecordInstance(ResourceRequest.class);
         .newRecordInstance(ResourceRequest.class);
     request.setPriority(r.getPriority());
     request.setPriority(r.getPriority());
-    request.setHostName(r.getHostName());
+    request.setResourceName(r.getResourceName());
     request.setCapability(r.getCapability());
     request.setCapability(r.getCapability());
     request.setNumContainers(r.getNumContainers());
     request.setNumContainers(r.getNumContainers());
     return request;
     return request;

+ 6 - 6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java

@@ -123,11 +123,11 @@ public class AppSchedulingInfo {
     // Update resource requests
     // Update resource requests
     for (ResourceRequest request : requests) {
     for (ResourceRequest request : requests) {
       Priority priority = request.getPriority();
       Priority priority = request.getPriority();
-      String hostName = request.getHostName();
+      String resourceName = request.getResourceName();
       boolean updatePendingResources = false;
       boolean updatePendingResources = false;
       ResourceRequest lastRequest = null;
       ResourceRequest lastRequest = null;
 
 
-      if (hostName.equals(ResourceRequest.ANY)) {
+      if (resourceName.equals(ResourceRequest.ANY)) {
         if (LOG.isDebugEnabled()) {
         if (LOG.isDebugEnabled()) {
           LOG.debug("update:" + " application=" + applicationId + " request="
           LOG.debug("update:" + " application=" + applicationId + " request="
               + request);
               + request);
@@ -152,10 +152,10 @@ public class AppSchedulingInfo {
         this.requests.put(priority, asks);
         this.requests.put(priority, asks);
         this.priorities.add(priority);
         this.priorities.add(priority);
       } else if (updatePendingResources) {
       } else if (updatePendingResources) {
-        lastRequest = asks.get(hostName);
+        lastRequest = asks.get(resourceName);
       }
       }
 
 
-      asks.put(hostName, request);
+      asks.put(resourceName, request);
       if (updatePendingResources) {
       if (updatePendingResources) {
         
         
         // Similarly, deactivate application?
         // Similarly, deactivate application?
@@ -187,9 +187,9 @@ public class AppSchedulingInfo {
   }
   }
 
 
   synchronized public ResourceRequest getResourceRequest(Priority priority,
   synchronized public ResourceRequest getResourceRequest(Priority priority,
-      String nodeAddress) {
+      String resourceName) {
     Map<String, ResourceRequest> nodeRequests = requests.get(priority);
     Map<String, ResourceRequest> nodeRequests = requests.get(priority);
-    return (nodeRequests == null) ? null : nodeRequests.get(nodeAddress);
+    return (nodeRequests == null) ? null : nodeRequests.get(resourceName);
   }
   }
 
 
   public synchronized Resource getResource(Priority priority) {
   public synchronized Resource getResource(Priority priority) {

+ 2 - 2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java

@@ -152,8 +152,8 @@ public class FiCaSchedulerApp extends SchedulerApplication {
     return this.appSchedulingInfo.getPriorities();
     return this.appSchedulingInfo.getPriorities();
   }
   }
 
 
-  public ResourceRequest getResourceRequest(Priority priority, String nodeAddress) {
-    return this.appSchedulingInfo.getResourceRequest(priority, nodeAddress);
+  public ResourceRequest getResourceRequest(Priority priority, String resourceName) {
+    return this.appSchedulingInfo.getResourceRequest(priority, resourceName);
   }
   }
 
 
   public synchronized int getTotalRequiredResources(Priority priority) {
   public synchronized int getTotalRequiredResources(Priority priority) {

+ 1 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java

@@ -130,7 +130,7 @@ public class MockAM {
   public ResourceRequest createResourceReq(String resource, int memory, int priority,
   public ResourceRequest createResourceReq(String resource, int memory, int priority,
       int containers) throws Exception {
       int containers) throws Exception {
     ResourceRequest req = Records.newRecord(ResourceRequest.class);
     ResourceRequest req = Records.newRecord(ResourceRequest.class);
-    req.setHostName(resource);
+    req.setResourceName(resource);
     req.setNumContainers(containers);
     req.setNumContainers(containers);
     Priority pri = Records.newRecord(Priority.class);
     Priority pri = Records.newRecord(Priority.class);
     pri.setPriority(priority);
     pri.setPriority(priority);

+ 1 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java

@@ -122,7 +122,7 @@ public class TestUtils {
     Resource capability = Resources.createResource(memory, 1);
     Resource capability = Resources.createResource(memory, 1);
     
     
     request.setNumContainers(numContainers);
     request.setNumContainers(numContainers);
-    request.setHostName(hostName);
+    request.setResourceName(hostName);
     request.setCapability(capability);
     request.setCapability(capability);
     request.setPriority(priority);
     request.setPriority(priority);
     return request;
     return request;

+ 1 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java

@@ -153,7 +153,7 @@ public class TestFairScheduler {
       int priority, int numContainers, boolean relaxLocality) {
       int priority, int numContainers, boolean relaxLocality) {
     ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class);
     ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class);
     request.setCapability(Resources.createResource(memory));
     request.setCapability(Resources.createResource(memory));
-    request.setHostName(host);
+    request.setResourceName(host);
     request.setNumContainers(numContainers);
     request.setNumContainers(numContainers);
     Priority prio = recordFactory.newRecordInstance(Priority.class);
     Priority prio = recordFactory.newRecordInstance(Priority.class);
     prio.setPriority(priority);
     prio.setPriority(priority);

+ 1 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java

@@ -106,7 +106,7 @@ public class TestFifoScheduler {
     ResourceRequest request = recordFactory
     ResourceRequest request = recordFactory
         .newRecordInstance(ResourceRequest.class);
         .newRecordInstance(ResourceRequest.class);
     request.setCapability(Resources.createResource(memory));
     request.setCapability(Resources.createResource(memory));
-    request.setHostName(host);
+    request.setResourceName(host);
     request.setNumContainers(numContainers);
     request.setNumContainers(numContainers);
     Priority prio = recordFactory.newRecordInstance(Priority.class);
     Priority prio = recordFactory.newRecordInstance(Priority.class);
     prio.setPriority(priority);
     prio.setPriority(priority);