MAPREDUCE-5487. In task processes, JobConf is unnecessarily loaded again in Limits (Sandy Ryza)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1524417 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -27,6 +27,9 @@ Release 2.3.0 - UNRELEASED
 
     MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza)
 
+    MAPREDUCE-5487. In task processes, JobConf is unnecessarily loaded again
+    in Limits (Sandy Ryza)
+
   BUG FIXES
 
     MAPREDUCE-5316. job -list-attempt-ids command does not handle illegal

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java

@@ -41,6 +41,7 @@ import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.counters.Limits;
 import org.apache.hadoop.mapreduce.filecache.DistributedCache;
 import org.apache.hadoop.mapreduce.security.TokenCache;
 import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
@@ -76,6 +77,8 @@ class YarnChild {
     LOG.debug("Child starting");
 
     final JobConf job = new JobConf();
+    // Initing with our JobConf allows us to avoid loading confs twice
+    Limits.init(job);
     job.addResource(MRJobConfig.JOB_CONF_FILE);
     UserGroupInformation.setConfiguration(job);
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java

@@ -62,8 +62,8 @@ import com.google.common.collect.Iterators;
 public class Counters
     extends AbstractCounters<Counters.Counter, Counters.Group> {
   
-  public static int MAX_COUNTER_LIMIT = Limits.COUNTERS_MAX;
-  public static int MAX_GROUP_LIMIT = Limits.GROUPS_MAX;
+  public static int MAX_COUNTER_LIMIT = Limits.getCountersMax();
+  public static int MAX_GROUP_LIMIT = Limits.getGroupsMax();
   private static HashMap<String, String> depricatedCounterMap =
       new HashMap<String, String>();
   

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java

@@ -28,37 +28,80 @@ import static org.apache.hadoop.mapreduce.MRJobConfig.*;
 public class Limits {
 
   static final Configuration conf = new JobConf();
-  public static final int GROUP_NAME_MAX =
-      conf.getInt(COUNTER_GROUP_NAME_MAX_KEY, COUNTER_GROUP_NAME_MAX_DEFAULT);
-  public static final int COUNTER_NAME_MAX =
-      conf.getInt(COUNTER_NAME_MAX_KEY, COUNTER_NAME_MAX_DEFAULT);
-  public static final int GROUPS_MAX =
-      conf.getInt(COUNTER_GROUPS_MAX_KEY, COUNTER_GROUPS_MAX_DEFAULT);
-  public static final int COUNTERS_MAX =
-      conf.getInt(COUNTERS_MAX_KEY, COUNTERS_MAX_DEFAULT);
 
   private int totalCounters;
   private LimitExceededException firstViolation;
 
+  private static boolean isInited;
+  
+  private static int GROUP_NAME_MAX;
+  private static int COUNTER_NAME_MAX;
+  private static int GROUPS_MAX;
+  private static int COUNTERS_MAX;
+  
+  public synchronized static void init(Configuration conf) {
+    if (!isInited) {
+      if (conf == null) {
+        conf = new JobConf();
+      }
+      GROUP_NAME_MAX = conf.getInt(COUNTER_GROUP_NAME_MAX_KEY,
+          COUNTER_GROUP_NAME_MAX_DEFAULT);
+      COUNTER_NAME_MAX = conf.getInt(COUNTER_NAME_MAX_KEY,
+          COUNTER_NAME_MAX_DEFAULT);
+      GROUPS_MAX = conf.getInt(COUNTER_GROUPS_MAX_KEY, COUNTER_GROUPS_MAX_DEFAULT);
+      COUNTERS_MAX = conf.getInt(COUNTERS_MAX_KEY, COUNTERS_MAX_DEFAULT);
+    }
+    isInited = true;
+  }
+  
+  public static int getGroupNameMax() {
+    if (!isInited) {
+      init(null);
+    }
+    return GROUP_NAME_MAX;
+  }
+  
+  public static int getCounterNameMax() {
+    if (!isInited) {
+      init(null);
+    }
+    return COUNTER_NAME_MAX;
+  }
+  
+  public static int getGroupsMax() {
+    if (!isInited) {
+      init(null);
+    }
+    return GROUPS_MAX;
+  }
+  
+  public static int getCountersMax() {
+    if (!isInited) {
+      init(null);
+    }
+    return COUNTERS_MAX;
+  }
+  
   public static String filterName(String name, int maxLen) {
     return name.length() > maxLen ? name.substring(0, maxLen - 1) : name;
   }
 
   public static String filterCounterName(String name) {
-    return filterName(name, COUNTER_NAME_MAX);
+    return filterName(name, getCounterNameMax());
   }
 
   public static String filterGroupName(String name) {
-    return filterName(name, GROUP_NAME_MAX);
+    return filterName(name, getGroupNameMax());
   }
 
   public synchronized void checkCounters(int size) {
     if (firstViolation != null) {
       throw new LimitExceededException(firstViolation);
     }
-    if (size > COUNTERS_MAX) {
+    int countersMax = getCountersMax();
+    if (size > countersMax) {
       firstViolation = new LimitExceededException("Too many counters: "+ size +
-                                                  " max="+ COUNTERS_MAX);
+                                                  " max="+ countersMax);
       throw firstViolation;
     }
   }
@@ -72,9 +115,10 @@ public class Limits {
     if (firstViolation != null) {
       throw new LimitExceededException(firstViolation);
     }
-    if (size > GROUPS_MAX) {
+    int groupsMax = getGroupsMax();
+    if (size > groupsMax) {
       firstViolation = new LimitExceededException("Too many counter groups: "+
-                                                  size +" max="+ GROUPS_MAX);
+                                                  size +" max="+ groupsMax);
     }
   }
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java

@@ -101,8 +101,8 @@ public class TestCounters {
   static final long FS_COUNTER_VALUE = 10;
 
   private void testMaxCounters(final Counters counters) {
-    LOG.info("counters max="+ Limits.COUNTERS_MAX);
-    for (int i = 0; i < Limits.COUNTERS_MAX; ++i) {
+    LOG.info("counters max="+ Limits.getCountersMax());
+    for (int i = 0; i < Limits.getCountersMax(); ++i) {
       counters.findCounter("test", "test"+ i);
     }
     setExpected(counters);
@@ -115,8 +115,8 @@ public class TestCounters {
   }
 
   private void testMaxGroups(final Counters counters) {
-    LOG.info("counter groups max="+ Limits.GROUPS_MAX);
-    for (int i = 0; i < Limits.GROUPS_MAX; ++i) {
+    LOG.info("counter groups max="+ Limits.getGroupsMax());
+    for (int i = 0; i < Limits.getGroupsMax(); ++i) {
       // assuming COUNTERS_MAX > GROUPS_MAX
       counters.findCounter("test"+ i, "test");
     }