HADOOP-7154. Set MALLOC_ARENA_MAX in hadoop-env.sh. Contributed by Todd Lipcon.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1@1364928 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -230,6 +230,9 @@ Release 1.1.0 - unreleased
     HDFS-2465. Add HDFS support for fadvise readahead and drop-behind.
     (todd, backported by Brandon Li via suresh)
 
+    HADOOP-7154. Should set MALLOC_ARENA_MAX in hadoop-env.sh
+    (todd via mattf)
+
     MAPREDUCE-4399. Change the Jetty response buffer size to improve 
     shuffle performance. (Luke Lu via suresh)
 

bin/hadoop-config.sh

@@ -75,6 +75,11 @@ if [ "$HADOOP_HOME_WARN_SUPPRESS" = "" ] && [ "$HADOOP_HOME" != "" ]; then
   echo 1>&2
 fi
 
+# Newer versions of glibc use an arena memory allocator that causes virtual
+# memory usage to explode. This interacts badly with the many threads that
+# we use in Hadoop. Tune the variable down to prevent vmem explosion.
+export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4}
+
 export HADOOP_HOME=${HADOOP_PREFIX}
 export HADOOP_HOME_WARN_SUPPRESS=1