Selaa lähdekoodia

YARN-2229. Changed the integer field of ContainerId to be long type. Contributed by Tsuyoshi OZAWA

Jian He 10 vuotta sitten
vanhempi
commit
3122daa802
25 muutettua tiedostoa jossa 114 lisäystä ja 76 poistoa
  1. 1 1
      hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/CheckpointAMPreemptionPolicy.java
  2. 3 0
      hadoop-yarn-project/CHANGES.txt
  3. 29 19
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
  4. 1 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
  5. 7 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java
  6. 16 8
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java
  7. 16 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
  8. 2 2
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
  9. 1 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
  10. 3 3
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
  11. 1 2
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
  12. 2 2
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
  13. 3 3
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
  14. 2 2
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
  15. 1 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
  16. 2 2
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
  17. 4 7
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/Epoch.java
  18. 3 3
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/EpochPBImpl.java
  19. 6 8
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
  20. 1 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
  21. 1 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
  22. 3 3
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
  23. 2 2
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
  24. 3 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
  25. 1 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java

+ 1 - 1
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/CheckpointAMPreemptionPolicy.java

@@ -194,7 +194,7 @@ public class CheckpointAMPreemptionPolicy implements AMPreemptionPolicy {
           Collections.sort(listOfCont, new Comparator<Container>() {
             @Override
             public int compare(final Container o1, final Container o2) {
-              return o2.getId().getId() - o1.getId().getId();
+              return o2.getId().compareTo(o1.getId());
             }
           });
 

+ 3 - 0
hadoop-yarn-project/CHANGES.txt

@@ -211,6 +211,9 @@ Release 2.6.0 - UNRELEASED
     YARN-2538. Added logs when RM sends roll-overed AMRMToken to AM. (Xuan Gong
     via zjshen)
 
+    YARN-2229. Changed the integer field of ContainerId to be long type.
+    (Tsuyoshi OZAWA via jianhe)
+
   OPTIMIZATIONS
 
   BUG FIXES

+ 29 - 19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java

@@ -41,9 +41,9 @@ public abstract class ContainerId implements Comparable<ContainerId>{
   @Private
   @Unstable
   public static ContainerId newInstance(ApplicationAttemptId appAttemptId,
-      int containerId) {
+      long containerId) {
     ContainerId id = Records.newRecord(ContainerId.class);
-    id.setId(containerId);
+    id.setContainerId(containerId);
     id.setApplicationAttemptId(appAttemptId);
     id.build();
     return id;
@@ -74,16 +74,28 @@ public abstract class ContainerId implements Comparable<ContainerId>{
   protected abstract void setApplicationAttemptId(ApplicationAttemptId atId);
 
   /**
-   * Get the identifier of the <code>ContainerId</code>.
-   * @return identifier of the <code>ContainerId</code>
+   * Get the lower 32 bits of identifier of the <code>ContainerId</code>,
+   * which doesn't include epoch. Note that this method will be marked as
+   * deprecated, so please use <code>getContainerId</code> instead.
+   * @return lower 32 bits of identifier of the <code>ContainerId</code>
    */
   @Public
   @Stable
   public abstract int getId();
 
+  /**
+   * Get the identifier of the <code>ContainerId</code>. Upper 24 bits are
+   * reserved as epoch of cluster, and lower 40 bits are reserved as
+   * sequential number of containers.
+   * @return identifier of the <code>ContainerId</code>
+   */
+  @Public
+  @Unstable
+  public abstract long getContainerId();
+
   @Private
   @Unstable
-  protected abstract void setId(int id);
+  protected abstract void setContainerId(long id);
  
   
   // TODO: fail the app submission if attempts are more than 10 or something
@@ -109,14 +121,12 @@ public abstract class ContainerId implements Comparable<ContainerId>{
           return fmt;
         }
       };
-  
+
   @Override
   public int hashCode() {
-    // Generated by eclipse.
-    final int prime = 435569;
-    int result = 7507;
-    result = prime * result + getId();
-    result = prime * result + getApplicationAttemptId().hashCode();
+    // Generated by IntelliJ IDEA 13.1.
+    int result = (int) (getContainerId() ^ (getContainerId() >>> 32));
+    result = 31 * result + getApplicationAttemptId().hashCode();
     return result;
   }
 
@@ -131,7 +141,7 @@ public abstract class ContainerId implements Comparable<ContainerId>{
     ContainerId other = (ContainerId) obj;
     if (!this.getApplicationAttemptId().equals(other.getApplicationAttemptId()))
       return false;
-    if (this.getId() != other.getId())
+    if (this.getContainerId() != other.getContainerId())
       return false;
     return true;
   }
@@ -140,12 +150,12 @@ public abstract class ContainerId implements Comparable<ContainerId>{
   public int compareTo(ContainerId other) {
     if (this.getApplicationAttemptId().compareTo(
         other.getApplicationAttemptId()) == 0) {
-      return this.getId() - other.getId();
+      return Long.valueOf(getContainerId())
+          .compareTo(Long.valueOf(other.getContainerId()));
     } else {
       return this.getApplicationAttemptId().compareTo(
           other.getApplicationAttemptId());
     }
-    
   }
 
   @Override
@@ -159,8 +169,8 @@ public abstract class ContainerId implements Comparable<ContainerId>{
     sb.append(
         appAttemptIdAndEpochFormat.get().format(
             getApplicationAttemptId().getAttemptId())).append("_");
-    sb.append(containerIdFormat.get().format(0x3fffff & getId()));
-    int epoch = getId() >> 22;
+    sb.append(containerIdFormat.get().format(0xffffffffffL & getContainerId()));
+    long epoch = getContainerId() >> 40;
     if (epoch > 0) {
       sb.append("_").append(appAttemptIdAndEpochFormat.get().format(epoch));
     }
@@ -177,12 +187,12 @@ public abstract class ContainerId implements Comparable<ContainerId>{
     }
     try {
       ApplicationAttemptId appAttemptID = toApplicationAttemptId(it);
-      int id = Integer.parseInt(it.next());
-      int epoch = 0;
+      long id = Long.parseLong(it.next());
+      long epoch = 0;
       if (it.hasNext()) {
         epoch = Integer.parseInt(it.next());
       }
-      int cid = (epoch << 22) | id;
+      long cid = (epoch << 40) | id;
       ContainerId containerId = ContainerId.newInstance(appAttemptID, cid);
       return containerId;
     } catch (NumberFormatException n) {

+ 1 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto

@@ -50,7 +50,7 @@ message ApplicationAttemptIdProto {
 message ContainerIdProto {
   optional ApplicationIdProto app_id = 1;
   optional ApplicationAttemptIdProto app_attempt_id = 2;
-  optional int32 id = 3;
+  optional int64 id = 3;
 }
 
 message ResourceProto {

+ 7 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java

@@ -49,12 +49,18 @@ public class ContainerIdPBImpl extends ContainerId {
 
   @Override
   public int getId() {
+    Preconditions.checkNotNull(proto);
+    return (int) proto.getId();
+  }
+
+  @Override
+  public long getContainerId() {
     Preconditions.checkNotNull(proto);
     return proto.getId();
   }
 
   @Override
-  protected void setId(int id) {
+  protected void setContainerId(long id) {
     Preconditions.checkNotNull(builder);
     builder.setId((id));
   }

+ 16 - 8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java

@@ -54,18 +54,26 @@ public class TestContainerId {
     long ts = System.currentTimeMillis();
     ContainerId c6 = newContainerId(36473, 4365472, ts, 25645811);
     Assert.assertEquals("container_10_0001_01_000001", c1.toString());
-    Assert.assertEquals(c1,
-        ContainerId.fromString("container_10_0001_01_000001"));
-    Assert.assertEquals(479987, 0x003fffff & c6.getId());
-    Assert.assertEquals(6, c6.getId() >> 22);
-    Assert.assertEquals("container_" + ts + "_36473_4365472_479987_06",
+    Assert.assertEquals(25645811, 0xffffffffffL & c6.getContainerId());
+    Assert.assertEquals(0, c6.getContainerId() >> 40);
+    Assert.assertEquals("container_" + ts + "_36473_4365472_25645811",
         c6.toString());
-    Assert.assertEquals(c6,
-        ContainerId.fromString("container_" + ts + "_36473_4365472_479987_06"));
+
+    ContainerId c7 = newContainerId(36473, 4365472, ts, 4298334883325L);
+    Assert.assertEquals(999799999997L, 0xffffffffffL & c7.getContainerId());
+    Assert.assertEquals(3, c7.getContainerId() >> 40);
+    Assert.assertEquals(
+        "container_" + ts + "_36473_4365472_999799999997_03", c7.toString());
+
+    ContainerId c8 = newContainerId(36473, 4365472, ts, 844424930131965L);
+    Assert.assertEquals(1099511627773L, 0xffffffffffL & c8.getContainerId());
+    Assert.assertEquals(767, c8.getContainerId() >> 40);
+    Assert.assertEquals(
+        "container_" + ts + "_36473_4365472_1099511627773_767", c8.toString());
   }
 
   public static ContainerId newContainerId(int appId, int appAttemptId,
-      long timestamp, int containerId) {
+      long timestamp, long containerId) {
     ApplicationId applicationId = ApplicationId.newInstance(timestamp, appId);
     ApplicationAttemptId applicationAttemptId =
         ApplicationAttemptId.newInstance(applicationId, appAttemptId);

+ 16 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java

@@ -59,9 +59,24 @@ public class TestConverterUtils {
   public void testContainerIdWithEpoch() throws URISyntaxException {
     ContainerId id = TestContainerId.newContainerId(0, 0, 0, 25645811);
     String cid = ConverterUtils.toString(id);
-    assertEquals("container_0_0000_00_479987_06", cid);
+    assertEquals("container_0_0000_00_25645811", cid);
     ContainerId gen = ConverterUtils.toContainerId(cid);
     assertEquals(gen.toString(), id.toString());
+
+    long ts = System.currentTimeMillis();
+    ContainerId id2 =
+        TestContainerId.newContainerId(36473, 4365472, ts, 4298334883325L);
+    String cid2 = ConverterUtils.toString(id2);
+    assertEquals("container_" + ts + "_36473_4365472_999799999997_03", cid2);
+    ContainerId gen2 = ConverterUtils.toContainerId(cid2);
+    assertEquals(gen2.toString(), id2.toString());
+
+    ContainerId id3 =
+        TestContainerId.newContainerId(36473, 4365472, ts, 844424930131965L);
+    String cid3 = ConverterUtils.toString(id3);
+    assertEquals("container_" + ts + "_36473_4365472_1099511627773_767", cid3);
+    ContainerId gen3 = ConverterUtils.toContainerId(cid3);
+    assertEquals(gen3.toString(), id3.toString());
   }
 
   @Test

+ 2 - 2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java

@@ -138,12 +138,12 @@ public class BuilderUtils {
   }
 
   public static ContainerId newContainerId(ApplicationAttemptId appAttemptId,
-      int containerId) {
+      long containerId) {
     return ContainerId.newInstance(appAttemptId, containerId);
   }
 
   public static ContainerId newContainerId(int appId, int appAttemptId,
-      long timestamp, int id) {
+      long timestamp, long id) {
     ApplicationId applicationId = newApplicationId(timestamp, appId);
     ApplicationAttemptId applicationAttemptId = newApplicationAttemptId(
         applicationId, appAttemptId);

+ 1 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java

@@ -107,5 +107,5 @@ public interface RMContext {
 
   boolean isWorkPreservingRecoveryEnabled();
   
-  int getEpoch();
+  long getEpoch();
 }

+ 3 - 3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java

@@ -84,7 +84,7 @@ public class RMContextImpl implements RMContext {
   private RMApplicationHistoryWriter rmApplicationHistoryWriter;
   private SystemMetricsPublisher systemMetricsPublisher;
   private ConfigurationProvider configurationProvider;
-  private int epoch;
+  private long epoch;
 
   /**
    * Default constructor. To be used in conjunction with setter methods for
@@ -375,11 +375,11 @@ public class RMContextImpl implements RMContext {
   }
 
   @Override
-  public int getEpoch() {
+  public long getEpoch() {
     return this.epoch;
   }
 
- void setEpoch(int epoch) {
+ void setEpoch(long epoch) {
     this.epoch = epoch;
   }
 }

+ 1 - 2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java

@@ -602,8 +602,7 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
         if (priorityComp != 0) {
           return priorityComp;
         }
-        return b.getContainerId().getId() -
-               a.getContainerId().getId();
+        return b.getContainerId().compareTo(a.getContainerId());
       }
     });
   }

+ 2 - 2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java

@@ -165,9 +165,9 @@ public class FileSystemRMStateStore extends RMStateStore {
   }
   
   @Override
-  public synchronized int getAndIncrementEpoch() throws Exception {
+  public synchronized long getAndIncrementEpoch() throws Exception {
     Path epochNodePath = getNodePath(rootDirPath, EPOCH_NODE);
-    int currentEpoch = 0;
+    long currentEpoch = 0;
     if (fs.exists(epochNodePath)) {
       // load current epoch
       FileStatus status = fs.getFileStatus(epochNodePath);

+ 3 - 3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java

@@ -44,7 +44,7 @@ import com.google.common.annotations.VisibleForTesting;
 public class MemoryRMStateStore extends RMStateStore {
   
   RMState state = new RMState();
-  private int epoch = 0;
+  private long epoch = 0L;
   
   @VisibleForTesting
   public RMState getState() {
@@ -56,8 +56,8 @@ public class MemoryRMStateStore extends RMStateStore {
   }
 
   @Override
-  public synchronized int getAndIncrementEpoch() throws Exception {
-    int currentEpoch = epoch;
+  public synchronized long getAndIncrementEpoch() throws Exception {
+    long currentEpoch = epoch;
     epoch = epoch + 1;
     return currentEpoch;
   }

+ 2 - 2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java

@@ -49,8 +49,8 @@ public class NullRMStateStore extends RMStateStore {
   }
 
   @Override
-  public synchronized int getAndIncrementEpoch() throws Exception {
-    return 0;
+  public synchronized long getAndIncrementEpoch() throws Exception {
+    return 0L;
   }
 
   @Override

+ 1 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java

@@ -545,7 +545,7 @@ public abstract class RMStateStore extends AbstractService {
   /**
    * Get the current epoch of RM and increment the value.
    */
-  public abstract int getAndIncrementEpoch() throws Exception;
+  public abstract long getAndIncrementEpoch() throws Exception;
   
   /**
    * Blocking API

+ 2 - 2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java

@@ -412,9 +412,9 @@ public class ZKRMStateStore extends RMStateStore {
   }
 
   @Override
-  public synchronized int getAndIncrementEpoch() throws Exception {
+  public synchronized long getAndIncrementEpoch() throws Exception {
     String epochNodePath = getNodePath(zkRootNodePath, EPOCH_NODE);
-    int currentEpoch = 0;
+    long currentEpoch = 0;
     if (existsWithRetries(epochNodePath, true) != null) {
       // load current epoch
       byte[] data = getDataWithRetries(epochNodePath, true);

+ 4 - 7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/Epoch.java

@@ -32,15 +32,15 @@ import org.apache.hadoop.yarn.util.Records;
 @Unstable
 public abstract class Epoch {
 
-  public static Epoch newInstance(int sequenceNumber) {
+  public static Epoch newInstance(long sequenceNumber) {
     Epoch epoch = Records.newRecord(Epoch.class);
     epoch.setEpoch(sequenceNumber);
     return epoch;
   }
 
-  public abstract int getEpoch();
+  public abstract long getEpoch();
 
-  public abstract void setEpoch(int sequenceNumber);
+  public abstract void setEpoch(long sequenceNumber);
 
   public abstract EpochProto getProto();
 
@@ -50,10 +50,7 @@ public abstract class Epoch {
 
   @Override
   public int hashCode() {
-    final int prime = 31;
-    int result = 1;
-    result = prime * result + getEpoch();
-    return result;
+    return (int) (getEpoch() ^ (getEpoch() >>> 32));
   }
 
   @Override

+ 3 - 3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/EpochPBImpl.java

@@ -53,13 +53,13 @@ public class EpochPBImpl extends Epoch {
   }
 
   @Override
-  public int getEpoch() {
+  public long getEpoch() {
     EpochProtoOrBuilder p = viaProto ? proto : builder;
-    return (int) (p.getEpoch() & 0xffffffff);
+    return p.getEpoch();
   }
 
   @Override
-  public void setEpoch(int sequentialNumber) {
+  public void setEpoch(long sequentialNumber) {
     maybeInitBuilder();
     builder.setEpoch(sequentialNumber);
   }

+ 6 - 8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java

@@ -26,7 +26,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -58,9 +58,8 @@ public class AppSchedulingInfo {
   Queue queue;
   final String user;
   // TODO making containerIdCounter long
-  private final AtomicInteger containerIdCounter;
-  private final int EPOCH_BIT_MASK = 0x3ff;
-  private final int EPOCH_BIT_SHIFT = 22;
+  private final AtomicLong containerIdCounter;
+  private final int EPOCH_BIT_SHIFT = 40;
 
   final Set<Priority> priorities = new TreeSet<Priority>(
       new org.apache.hadoop.yarn.server.resourcemanager.resource.Priority.Comparator());
@@ -77,15 +76,14 @@ public class AppSchedulingInfo {
  
   public AppSchedulingInfo(ApplicationAttemptId appAttemptId,
       String user, Queue queue, ActiveUsersManager activeUsersManager,
-      int epoch) {
+      long epoch) {
     this.applicationAttemptId = appAttemptId;
     this.applicationId = appAttemptId.getApplicationId();
     this.queue = queue;
     this.queueName = queue.getQueueName();
     this.user = user;
     this.activeUsersManager = activeUsersManager;
-    this.containerIdCounter = new AtomicInteger(
-        (epoch & EPOCH_BIT_MASK) << EPOCH_BIT_SHIFT);
+    this.containerIdCounter = new AtomicLong(epoch << EPOCH_BIT_SHIFT);
   }
 
   public ApplicationId getApplicationId() {
@@ -117,7 +115,7 @@ public class AppSchedulingInfo {
     LOG.info("Application " + applicationId + " requests cleared");
   }
 
-  public int getNewContainerId() {
+  public long getNewContainerId() {
     return this.containerIdCounter.incrementAndGet();
   }
 

+ 1 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java

@@ -182,7 +182,7 @@ public class SchedulerApplicationAttempt {
     return this.pendingRelease;
   }
 
-  public int getNewContainerId() {
+  public long getNewContainerId() {
     return appSchedulingInfo.getNewContainerId();
   }
 

+ 1 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java

@@ -238,7 +238,7 @@ public class TestWorkPreservingRMRestart {
     }
 
     // *********** check appSchedulingInfo state ***********
-    assertEquals((1 << 22) + 1, schedulerAttempt.getNewContainerId());
+    assertEquals((1L << 40) + 1L, schedulerAttempt.getNewContainerId());
   }
 
   private void checkCSQueue(MockRM rm,

+ 3 - 3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java

@@ -508,13 +508,13 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{
     RMStateStore store = stateStoreHelper.getRMStateStore();
     store.setRMDispatcher(new TestDispatcher());
     
-    int firstTimeEpoch = store.getAndIncrementEpoch();
+    long firstTimeEpoch = store.getAndIncrementEpoch();
     Assert.assertEquals(0, firstTimeEpoch);
     
-    int secondTimeEpoch = store.getAndIncrementEpoch();
+    long secondTimeEpoch = store.getAndIncrementEpoch();
     Assert.assertEquals(1, secondTimeEpoch);
     
-    int thirdTimeEpoch = store.getAndIncrementEpoch();
+    long thirdTimeEpoch = store.getAndIncrementEpoch();
     Assert.assertEquals(2, thirdTimeEpoch);
   }
 

+ 2 - 2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java

@@ -63,13 +63,13 @@ public class TestSchedulerApplicationAttempt {
 
     ApplicationAttemptId appAttId = createAppAttemptId(0, 0);
     RMContext rmContext = mock(RMContext.class);
-    when(rmContext.getEpoch()).thenReturn(3);
+    when(rmContext.getEpoch()).thenReturn(3L);
     SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId,
         user, oldQueue, oldQueue.getActiveUsersManager(), rmContext);
     oldMetrics.submitApp(user);
     
     // confirm that containerId is calculated based on epoch.
-    assertEquals(app.getNewContainerId(), 0x00c00001);
+    assertEquals(0x30000000001L, app.getNewContainerId());
     
     // Resource request
     Resource requestedResource = Resource.newInstance(1536, 2);

+ 3 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java

@@ -172,7 +172,9 @@ public class TestUtils {
     ContainerId containerId = mock(ContainerId.class);
     doReturn(application.getApplicationAttemptId()).
     when(containerId).getApplicationAttemptId();
-    doReturn(application.getNewContainerId()).when(containerId).getId();
+    long id = application.getNewContainerId();
+    doReturn((int)id).when(containerId).getId();
+    doReturn(id).when(containerId).getContainerId();
     return containerId;
   }
   

+ 1 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java

@@ -63,7 +63,7 @@ public class TestMaxRunningAppsEnforcer {
     maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler);
     appNum = 0;
     rmContext = mock(RMContext.class);
-    when(rmContext.getEpoch()).thenReturn(0);
+    when(rmContext.getEpoch()).thenReturn(0L);
   }
   
   private FSAppAttempt addApp(FSLeafQueue queue, String user) {