MAPREDUCE-4379. Node Manager throws java.lang.OutOfMemoryError: Java heap space due to org.apache.hadoop.fs.LocalDirAllocator.contexts (Devaraj K via bobby)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1358305 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java

@@ -207,6 +207,19 @@ public class LocalDirAllocator {
       return contexts.containsKey(contextCfgItemName);
     }
   }
+  
+  /**
+   * Removes the context from the context config items
+   * 
+   * @param contextCfgItemName
+   */
+  @Deprecated
+  @InterfaceAudience.LimitedPrivate({"MapReduce"})
+  public static void removeContext(String contextCfgItemName) {
+    synchronized (contexts) {
+      contexts.remove(contextCfgItemName);
+    }
+  }
     
   /** We search through all the configured dirs for the file's existence
    *  and return true when we find  

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java

@@ -339,5 +339,18 @@ public class TestLocalDirAllocator {
     }
 
   }
+  
+  @Test
+  public void testRemoveContext() throws IOException {
+    String dir = buildBufferDir(ROOT, 0);
+    String contextCfgItemName = "application_1340842292563_0004.app.cache.dirs";
+    conf.set(contextCfgItemName, dir);
+    LocalDirAllocator localDirAllocator = new LocalDirAllocator(
+        contextCfgItemName);
+    localDirAllocator.getLocalPathForWrite("p1/x", SMALL_FILE_SIZE, conf);
+    assertTrue(LocalDirAllocator.isContextValid(contextCfgItemName));
+    LocalDirAllocator.removeContext(contextCfgItemName);
+    assertFalse(LocalDirAllocator.isContextValid(contextCfgItemName));
+  }
 
 }

hadoop-mapreduce-project/CHANGES.txt

@@ -643,6 +643,10 @@ Release 0.23.3 - UNRELEASED
     MAPREDUCE-4387. RM gets fatal error and exits during TestRM 
     (Kihwal Lee via tgraves)
 
+    MAPREDUCE-4379. Node Manager throws java.lang.OutOfMemoryError: Java heap
+    space due to org.apache.hadoop.fs.LocalDirAllocator.contexts (Devaraj K
+    via bobby)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java

@@ -91,6 +91,7 @@ public class ContainerLocalizer {
   private final LocalDirAllocator userDirs;
   private final RecordFactory recordFactory;
   private final Map<LocalResource,Future<Path>> pendingResources;
+  private final String appCacheDirContextName;
 
   public ContainerLocalizer(FileContext lfs, String user, String appId,
       String localizerId, List<Path> localDirs,
@@ -108,10 +109,9 @@ public class ContainerLocalizer {
     this.localizerId = localizerId;
     this.recordFactory = recordFactory;
     this.conf = new Configuration();
-    this.appDirs =
-      new LocalDirAllocator(String.format(APPCACHE_CTXT_FMT, appId));
-    this.userDirs =
-      new LocalDirAllocator(String.format(USERCACHE_CTXT_FMT, appId));
+    this.appCacheDirContextName = String.format(APPCACHE_CTXT_FMT, appId);
+    this.appDirs = new LocalDirAllocator(appCacheDirContextName);
+    this.userDirs = new LocalDirAllocator(String.format(USERCACHE_CTXT_FMT, user));
     this.pendingResources = new HashMap<LocalResource,Future<Path>>();
   }
 
@@ -121,6 +121,7 @@ public class ContainerLocalizer {
       rpc.getProxy(LocalizationProtocol.class, nmAddr, conf);
   }
 
+  @SuppressWarnings("deprecation")
   public int runLocalization(final InetSocketAddress nmAddr)
       throws IOException, InterruptedException {
     // load credentials
@@ -177,6 +178,7 @@ public class ContainerLocalizer {
       if (exec != null) {
         exec.shutdownNow();
       }
+      LocalDirAllocator.removeContext(appCacheDirContextName);
     }
   }
 
@@ -373,7 +375,7 @@ public class ContainerLocalizer {
       lfs.mkdir(appFileCacheDir, null, false);
     }
     conf.setStrings(String.format(APPCACHE_CTXT_FMT, appId), appsFileCacheDirs);
-    conf.setStrings(String.format(USERCACHE_CTXT_FMT, appId), usersFileCacheDirs);
+    conf.setStrings(String.format(USERCACHE_CTXT_FMT, user), usersFileCacheDirs);
   }
 
 }